)>}]
شركة التطبيقات المتكاملة لتصميم وبرمجة البرمجيات الخاصة ش.ش.و.
Integrated Applications Programming Company
Home » Code Library » Default (Ia.Statistics.Cl.Models.Natureland)

Public general use code classes and xml files that we've compiled and used over the years:

Structure of the natureland.net website.

    1: using System;
    2: using System.Collections.Generic;
    3: using System.IO;
    4: using System.Linq;
    5: using System.Reflection;
    6: using System.Xml.Linq;
    7: using System.Text.RegularExpressions;
    8: using System.Text;
    9: using OpenQA.Selenium;
   10: using OpenQA.Selenium.Chrome;
   11:  
   12: namespace Ia.Statistics.Cl.Models.Natureland
   13: {
   14:     ////////////////////////////////////////////////////////////////////////////
   15:  
   16:     /// <summary publish="true">
   17:     /// Structure of the natureland.net website.
   18:     /// </summary>
   19:     /// <remarks> 
   20:     /// Copyright © 2024-2025 Jasem Y. Al-Shamlan (info@ia.com.kw), Integrated Applications - Kuwait. All Rights Reserved.
   21:     ///
   22:     /// This library is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by
   23:     /// the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
   24:     ///
   25:     /// This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
   26:     /// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
   27:     /// 
   28:     /// You should have received a copy of the GNU General Public License along with this library. If not, see http://www.gnu.org/licenses.
   29:     /// 
   30:     /// Copyright notice: This notice may not be removed or altered from any source distribution.
   31:     /// </remarks> 
   32:     public class Default
   33:     {
   34:         private static XDocument xd;
   35:  
   36:         private static List<Ia.Statistics.Cl.Models.Category> categoryList;
   37:  
   38:         private static List<Ia.Statistics.Cl.Models.Product> productList;
   39:  
   40:         private static List<Ia.Statistics.Cl.Models.Site> naturelandSiteList = new List<Site>{
   41:             Ia.Statistics.Cl.Models.Site.NaturelandKuwait,
   42:             Ia.Statistics.Cl.Models.Site.NaturelandSaudiArabia,
   43:             Ia.Statistics.Cl.Models.Site.NaturelandEmirates,
   44:             Ia.Statistics.Cl.Models.Site.NaturelandQatar,
   45:             Ia.Statistics.Cl.Models.Site.NaturelandBahrain,
   46:         };
   47:  
   48:         ////////////////////////////////////////////////////////////////////////////
   49:  
   50:         /// <summary>
   51:         /// 
   52:         /// </summary>
   53:         public Default() { }
   54:  
   55:         ////////////////////////////////////////////////////////////////////////////
   56:  
   57:         /// <summary>
   58:         ///
   59:         /// </summary>
   60:         public static string ReadUpdateProductListOfCategorySequentially(Ia.Statistics.Cl.Models.Site site0)
   61:         {
   62:             string s;
   63:  
   64:             var category = Ia.Statistics.Cl.Models.Natureland.Default.CategoryList.RandomThenNext();
   65:  
   66:             // KW
   67:             //var category = (from c in Ia.Statistics.Cl.Model.Natureland.Default.CategoryList where c.Id == 20306 select c).SingleOrDefault();
   68:  
   69:             // SA
   70:             //var category = (from c in Ia.Statistics.Cl.Model.Natureland.Default.CategoryList where c.Id == 30503 select c).SingleOrDefault();
   71:  
   72:             var site = Ia.Statistics.Cl.Models.Site.SiteFromId(category.SiteId);
   73:  
   74:             var productPriceStockQuantitySoldList = ProductPriceStockListOfCategory(site, category, out string exception);
   75:  
   76:             if (string.IsNullOrEmpty(exception))
   77:             {
   78:                 if (productPriceStockQuantitySoldList.Count > 0)
   79:                 {
   80:                     s = string.Empty;
   81:  
   82:                     foreach (var productPriceStockQuantitySold in productPriceStockQuantitySoldList)
   83:                     {
   84:                         if (!string.IsNullOrEmpty(productPriceStockQuantitySold.Product.Sku))
   85:                         {
   86:                             Ia.Statistics.Cl.Models.Product.Create(productPriceStockQuantitySold.Product);
   87:                             Ia.Statistics.Cl.Models.ProductPriceSpot.Create(productPriceStockQuantitySold.Product, productPriceStockQuantitySold.Price);
   88:                             Ia.Statistics.Cl.Models.ProductStockSpot.Create(productPriceStockQuantitySold.Product, productPriceStockQuantitySold.Stock);
   89:  
   90:                             s += "Product: " + productPriceStockQuantitySold.Product.Name + "; price: " + productPriceStockQuantitySold.Price + "; stock: " + productPriceStockQuantitySold.Stock + ". ";
   91:                         }
   92:                         else
   93:                         {
   94:                             s += "Product: " + productPriceStockQuantitySold.Product.Name + ": SKU is null. ";
   95:                         }
   96:                     }
   97:                 }
   98:                 else s = "Product list is 0. ";
   99:  
  100:                 s = site.Name + ": " + category.Url + ": " + s;
  101:             }
  102:             else s = exception;
  103:  
  104:             return s;
  105:         }
  106:  
  107:         ////////////////////////////////////////////////////////////////////////////
  108:  
  109:         /// <summary>
  110:         ///
  111:         /// </summary>
  112:         public static List<Ia.Statistics.Cl.Models.ProductPriceStockQuantitySold> ProductPriceStockListOfCategory(Ia.Statistics.Cl.Models.Site site, Ia.Statistics.Cl.Models.Category category, out string exception)
  113:         {
  114:             int areaId;
  115:  
  116:             var productPriceStockQuantitySoldList = new List<Ia.Statistics.Cl.Models.ProductPriceStockQuantitySold>();
  117:  
  118:             var options = new ChromeOptions();
  119:  
  120:             var chromeDriverService = ChromeDriverService.CreateDefaultService();
  121:             chromeDriverService.HideCommandPromptWindow = true;
  122:             chromeDriverService.SuppressInitialDiagnosticInformation = true;
  123:  
  124:             options.AddArgument("--headless");
  125:             options.AddArgument("--log-level=3");
  126:             options.AddArgument("--disable-extensions");
  127:             options.AddArgument("--ignore-certificate-errors");
  128:  
  129:             switch (site.Iso)
  130:             {
  131:                 case "KW": areaId = 446; break;
  132:                 case "SA": areaId = 1563; break;
  133:                 case "AE": areaId = 1019; break;
  134:                 case "QA": areaId = 11; break;
  135:                 case "BH": areaId = 13; break;
  136:                 default: areaId = 446; break;
  137:             }
  138:  
  139:             using (var chromeDriver = new OpenQA.Selenium.Chrome.ChromeDriver(chromeDriverService, options))
  140:             {
  141:                 chromeDriver.Manage().Timeouts().ImplicitWait = Ia.Statistics.Cl.Models.Default.TimeoutTimeSpan;
  142:  
  143:                 chromeDriver.Navigate().GoToUrl(category.Url);
  144:  
  145:                 chromeDriver.Manage().Cookies.AddCookie(new OpenQA.Selenium.Cookie("1P_JAR", DateTime.UtcNow.AddHours(3).ToString("yyyy-MM-dd-hh")));
  146:                 chromeDriver.Manage().Cookies.AddCookie(new OpenQA.Selenium.Cookie("NAT_ACCESS_TOKEN", "7a8b9ea2f7.670fb5a00a5d4c525103156ed937c552b58dad8b6234083059d76c2fca1fa076"));
  147:                 chromeDriver.Manage().Cookies.AddCookie(new OpenQA.Selenium.Cookie("NAT_AREA", areaId.ToString()));
  148:                 chromeDriver.Manage().Cookies.AddCookie(new OpenQA.Selenium.Cookie("NAT_CART_ITEMS", "%7B%7D"));
  149:                 chromeDriver.Manage().Cookies.AddCookie(new OpenQA.Selenium.Cookie("NAT_COUNTRY", site.Iso));
  150:                 chromeDriver.Manage().Cookies.AddCookie(new OpenQA.Selenium.Cookie("NAT_LOCALE", "en"));
  151:                 chromeDriver.Manage().Cookies.AddCookie(new OpenQA.Selenium.Cookie("NAT_UID_KEY", "D03DEE19-B609-412E-D689-39A2B7ED1681"));
  152:                 chromeDriver.Manage().Cookies.AddCookie(new OpenQA.Selenium.Cookie("NEXT_LOCALE", "en"));
  153:  
  154:                 chromeDriver.Navigate().GoToUrl(category.Url); // I will access site again with location cookies
  155:  
  156:                 // maximize the browser
  157:                 // chromeDriver.Manage().Window.Maximize();
  158:  
  159:                 // scroll down the webpage
  160:                 var javaScriptExecutor = ((IJavaScriptExecutor)chromeDriver);
  161:                 javaScriptExecutor.ExecuteScript("window.scrollTo(0, document.body.scrollHeight)");
  162:  
  163:                 var content = chromeDriver.PageSource;
  164:  
  165:                 ProductPriceStockListFromCategoryContent(site, category, content, out productPriceStockQuantitySoldList, out exception);
  166:             }
  167:  
  168:             return productPriceStockQuantitySoldList;
  169:         }
  170:  
  171:         ////////////////////////////////////////////////////////////////////////////
  172:  
  173:         /// <summary>
  174:         ///
  175:         /// </summary>
  176:         public static void ProductPriceStockListFromCategoryContent(Ia.Statistics.Cl.Models.Site site, Ia.Statistics.Cl.Models.Category category, string content, out List<Ia.Statistics.Cl.Models.ProductPriceStockQuantitySold> productPriceStockQuantitySoldList, out string exception)
  177:         {
  178:             productPriceStockQuantitySoldList = new List<Ia.Statistics.Cl.Models.ProductPriceStockQuantitySold>();
  179:  
  180:             var match1 = Regex.Match(content, @"""locale"":""en-(.+?)""", RegexOptions.None, Ia.Statistics.Cl.Models.Default.TimeoutTimeSpan);
  181:  
  182:             if (match1.Success)
  183:             {
  184:                 var countryIso2 = match1.Groups[1].Value;
  185:  
  186:                 if (site.BaseUrl.Contains(countryIso2))
  187:                 {
  188:                     var match2 = Regex.Match(content, @"<script id=""__NEXT_DATA__"" type=""application/json"">(.+?)</script>", RegexOptions.None, Ia.Statistics.Cl.Models.Default.TimeoutTimeSpan);
  189:  
  190:                     if (match2.Success)
  191:                     {
  192:                         var s1 = match2.Groups[1].Value;
  193:  
  194:                         var match3 = Regex.Match(s1, @"""pageProps"":(\{""pageData"".+?\}\}),""pageFilters"":", RegexOptions.None, Ia.Statistics.Cl.Models.Default.TimeoutTimeSpan);
  195:  
  196:                         if (match3.Success)
  197:                         {
  198:                             var s2 = match3.Groups[1].Value + "}"; // important to fix JSON format of segment
  199:  
  200:                             // {"id":614,"name":"Cherry Tomatoes 250g","slug":"cherry-tomatoes-614","default_sku_id":611,"skus":[{"id":611,"name":"250g","slug":"250g-611","price":{"price":0.995,"old_price":null,"vat":false},"buy_limit":5,"medias":[{"url":"https://naturelandcdn-master.fra1.cdn.digitaloceanspaces.com/media/noname/sku/q5FnF9/cb1e8879-1820-4bf6-9f2c-ca3276cee639_izc9O8sjO.jpeg","type":"image","blurhash":"UYSYjk%Mpwx[%Mj@fkkCuibHVEoM%Mj[jFae","format":"jpeg"}]}],"brand":{"id":2,"name":"Earth","slug":"earth-2"},"category":{"id":1,"slug":"fruits-vegetables-1","name":"Fruits \u0026 Vegetables","color":"#334455","text_color":"#000000"},"subcategory":{"id":3,"name":"Vegetables","slug":"vegetables-3"}},
  201:                             var matchCollection = Regex.Matches(s2, @"{""id"":(.+?),""name"":""(.+?)"",""slug"":""(.+?)"",""default_sku_id"":(.+?),""skus"":.+?,""price"":{""price"":(.+?),.+?,""buy_limit"":(.+?),.+?}}", RegexOptions.Singleline, Ia.Statistics.Cl.Models.Default.TimeoutTimeSpan);
  202:  
  203:                             foreach (Match match in matchCollection)
  204:                             {
  205:                                 var product = new Ia.Statistics.Cl.Models.Product();
  206:                                 var productPriceStockQuantitySold = new Ia.Statistics.Cl.Models.ProductPriceStockQuantitySold();
  207:  
  208:                                 var sku = match.Groups[4].Value; // default_sku_id;
  209:  
  210:                                 var name = match.Groups[2].Value; // name
  211:                                 name = Ia.Statistics.Cl.Models.Default.NormalizeName(name);
  212:  
  213:                                 var url = match.Groups[3].Value; // slug;
  214:  
  215:                                 product.Id = Ia.Statistics.Cl.Models.Product.ProductId(site, sku);
  216:                                 product.SiteId = site.Id;
  217:                                 //product.Category = category;
  218:                                 product.Sku = sku;
  219:                                 product.Name = name;
  220:                                 product.Url = site.BaseUrl + "/product/" + url;
  221:  
  222:                                 var price = decimal.TryParse(match.Groups[5].Value, out decimal d) ? d : 0;
  223:  
  224:                                 var stock = int.Parse(match.Groups[6].Value); // buy_limit;
  225:                                 stock = stock <= Ia.Statistics.Cl.Models.Default.MaximumReasonableStockValue ? stock : Ia.Statistics.Cl.Models.Default.MaximumReasonableStockValue;
  226:  
  227:                                 productPriceStockQuantitySold.Product = product;
  228:                                 productPriceStockQuantitySold.Price = price;
  229:                                 productPriceStockQuantitySold.Stock = stock;
  230:  
  231:                                 productPriceStockQuantitySoldList.Add(productPriceStockQuantitySold);
  232:                             }
  233:  
  234:                             exception = string.Empty;
  235:                         }
  236:                         else
  237:                         {
  238:                             exception = @"Error: Could not match ""pageProps"":(\{ ""pageData"".+?\}\}),""pageFilters"":";
  239:                         }
  240:                     }
  241:                     else
  242:                     {
  243:                         exception = @"Error: Could not match <script id=\""__NEXT_DATA__\"" type=\""application/json\"">(.+?)</script>";
  244:                     }
  245:                 }
  246:                 else
  247:                 {
  248:                     exception = @"Error: content file country ISO2 does not match site BaseUrl. ";
  249:                 }
  250:             }
  251:             else
  252:             {
  253:                 exception = @"Error: Could not match code2 in content file. ";
  254:             }
  255:         }
  256:  
  257:         ////////////////////////////////////////////////////////////////////////////
  258:  
  259:         /// <summary>
  260:         ///
  261:         /// </summary>
  262:         public static List<Ia.Statistics.Cl.Models.Category> CategoryList
  263:         {
  264:             get
  265:             {
  266:                 if (categoryList == null || categoryList.Count == 0)
  267:                 {
  268:                     categoryList = new List<Models.Category>();
  269:  
  270:                     foreach (var site in naturelandSiteList)
  271:                     {
  272:                         categoryList.AddRange((from c in XDocument.Element("natureland").Descendants("category")
  273:                                                where !c.HasElements
  274:                                                select new Ia.Statistics.Cl.Models.Category
  275:                                                {
  276:                                                    Id = Ia.Statistics.Cl.Models.Category.CategoryId(site, int.Parse(c.Attribute("id").Value)),
  277:                                                    SiteId = site.Id,
  278:                                                    Url = site.BaseUrl + c.Attribute("url").Value,
  279:                                                    Name = c.Attribute("name").Value
  280:                                                }
  281:                         ).ToList<Ia.Statistics.Cl.Models.Category>());
  282:  
  283:                         categoryList = categoryList.Shuffle().ToList();
  284:                     }
  285:                 }
  286:  
  287:                 return categoryList;
  288:             }
  289:         }
  290:  
  291:         ////////////////////////////////////////////////////////////////////////////
  292:  
  293:         /// <summary>
  294:         /// 
  295:         /// How to embed and access resources by using Visual C# http://support.microsoft.com/kb/319292/en-us
  296:         /// 
  297:         /// 1. Change the "Build Action" property of your XML file from "Content" to "Embedded Resource".
  298:         /// 2. Add "using System.Reflection".
  299:         /// 3. Manifest resource stream will start with the project namespace, the location of XML file.
  300:         /// 
  301:         /// </summary>
  302:  
  303:         private static XDocument XDocument
  304:         {
  305:             get
  306:             {
  307:                 Assembly assembly;
  308:                 StreamReader streamReader;
  309:  
  310:                 xd = null;
  311:                 assembly = Assembly.GetExecutingAssembly();
  312:                 streamReader = new StreamReader(assembly.GetManifestResourceStream("Ia.Statistics.Cl.Models.natureland.net.default.xml"));
  313:  
  314:                 try
  315:                 {
  316:                     if (streamReader.Peek() != -1)
  317:                     {
  318:                         xd = System.Xml.Linq.XDocument.Load(streamReader);
  319:                     }
  320:                 }
  321:                 catch (Exception)
  322:                 {
  323:                 }
  324:                 finally
  325:                 {
  326:                 }
  327:  
  328:                 return xd;
  329:             }
  330:         }
  331:  
  332:         ////////////////////////////////////////////////////////////////////////////
  333:         ////////////////////////////////////////////////////////////////////////////
  334:     }
  335: }