switch from htmlAgilityPack to AngleSharp

Author: Koenari

NetStone/Definitions/Model/PagedDefinition.cs

@@ -1,5 +1,4 @@
 using Newtonsoft.Json;
-using Newtonsoft.Json.Linq;
 
 namespace NetStone.Definitions.Model;
 

NetStone/LodestoneClient.cs

@@ -2,7 +2,8 @@
 using System.Net;
 using System.Net.Http;
 using System.Threading.Tasks;
-using HtmlAgilityPack;
+using AngleSharp;
+using AngleSharp.Dom;
 using NetStone.Definitions;
 using NetStone.GameData;
 using NetStone.Model;
@@ -251,9 +252,11 @@ await GetParsed($"/lodestone/freecompany/{query.BuildQueryString()}&page={page}"
     /// <param name="agent">The user agent to use for the request.</param>
     /// <exception cref="HttpRequestException"> The request failed due to an underlying issue such as network connectivity, DNS failure, server certificate validation or timeout.</exception>
     /// <returns>The instantiated LodestoneParseable in case of success.</returns>
-    private async Task<T?> GetParsed<T>(string url, Func<HtmlNode, T?> createParseable,
+    private async Task<T?> GetParsed<T>(string url, Func<IElement, T?> createParseable,
         UserAgent agent = UserAgent.Desktop) where T : LodestoneParseable
     {
+        var config = Configuration.Default.WithDefaultLoader();
+        var context = BrowsingContext.New(config);
         var request = new HttpRequestMessage(HttpMethod.Get, url);
 
         switch (agent)
@@ -273,10 +276,9 @@ await GetParsed($"/lodestone/freecompany/{query.BuildQueryString()}&page={page}"
         if (response.StatusCode == HttpStatusCode.NotFound)
             return null;
 
-        var doc = new HtmlDocument();
-        doc.LoadHtml(await response.Content.ReadAsStringAsync());
-
-        return createParseable.Invoke(doc.DocumentNode);
+        var doc = await context.OpenAsync(async void (req) => req.Content(await response.Content.ReadAsStringAsync()));
+        
+        return createParseable.Invoke(doc.Body!);
     }
 
     /// <inheritdoc />

NetStone/Model/IPaginatedResult.cs

@@ -1,7 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.Threading.Tasks;
-using HtmlAgilityPack;
+using AngleSharp.Dom;
 using NetStone.Definitions.Model;
 using NetStone.Search;
 
@@ -39,7 +39,7 @@ public abstract class PaginatedIdResult<TPage, TEntry, TEntryDef>
                                                        where TEntryDef : PagedEntryDefinition
 {
     ///<inheritdoc />
-    protected PaginatedIdResult(HtmlNode rootNode, PagedDefinition<TEntryDef> pageDefinition, 
+    protected PaginatedIdResult(IElement rootNode, PagedDefinition<TEntryDef> pageDefinition, 
                                 Func<string, int, Task<TPage?>> nextPageFunc, string id) 
         : base(rootNode, pageDefinition, nextPageFunc, id)
     {
@@ -55,7 +55,7 @@ public abstract class PaginatedSearchResult<TPage, TEntry, TEntryDef, TQuery>
                                                        where TQuery : ISearchQuery
 {
     ///<inheritdoc />
-    protected PaginatedSearchResult(HtmlNode rootNode, PagedDefinition<TEntryDef> pageDefinition, 
+    protected PaginatedSearchResult(IElement rootNode, PagedDefinition<TEntryDef> pageDefinition, 
                                     Func<TQuery, int, Task<TPage?>> nextPageFunc, 
                                     TQuery query) 
         : base(rootNode, pageDefinition, nextPageFunc, query)
@@ -85,7 +85,7 @@ public abstract class PaginatedResult<TPage, TEntry, TEntryDef,TRequest> : Lodes
     /// <param name="pageDefinition">CSS definitions for the paginated type</param>
     /// <param name="nextPageFunc">Function to retrieve a page of this type</param>
     /// <param name="request">The input used to request further pages.</param>
-    protected PaginatedResult(HtmlNode rootNode, PagedDefinition<TEntryDef> pageDefinition,Func<TRequest, int, Task<TPage?>> nextPageFunc, TRequest request) : base(rootNode)
+    protected PaginatedResult(IElement rootNode, PagedDefinition<TEntryDef> pageDefinition,Func<TRequest, int, Task<TPage?>> nextPageFunc, TRequest request) : base(rootNode)
     {
         this.PageDefinition = pageDefinition;
         this.request = request;

NetStone/Model/LodestoneParseable.cs

@@ -1,11 +1,10 @@
-using HtmlAgilityPack;
-using HtmlAgilityPack.CssSelectors.NetCore;
-using NetStone.Definitions;
+using NetStone.Definitions;
 using NetStone.Definitions.Model;
 using System;
 using System.Linq;
 using System.Text.RegularExpressions;
 using System.Web;
+using AngleSharp.Dom;
 
 namespace NetStone.Model;
 
@@ -17,33 +16,33 @@ public abstract class LodestoneParseable
     /// <summary>
     /// The HTML document's root node.
     /// </summary>
-    protected readonly HtmlNode RootNode;
+    protected readonly IElement RootNode;
 
     /// <summary>
     /// Constructs an instance of parseable data for given node
     /// </summary>
     /// <param name="rootNode"></param>
-    protected LodestoneParseable(HtmlNode rootNode)
+    protected LodestoneParseable(IElement rootNode)
     {
         this.RootNode = rootNode;
     }
 
     /// <summary>
-    /// Query a <see cref="HtmlNode"/> via pack selector.
+    /// Query a <see cref="Node"/> via pack selector.
     /// </summary>
     /// <param name="pack">Definition of the node.</param>
     /// <returns>The needed node.</returns>
-    protected HtmlNode QueryNode(DefinitionsPack pack) => this.RootNode.QuerySelector(pack.Selector);
+    protected IElement? QueryNode(DefinitionsPack pack) => this.RootNode.QuerySelector(pack.Selector);
 
     /// <summary>
-    /// Query all ChildNodes of a <see cref="HtmlNode"/> via pack selector.
+    /// Query all ChildNodes of a <see cref="Node"/> via pack selector.
     /// Removes unneeded "#text" nodes.
     /// </summary>
     /// <param name="pack">Definition of the node.</param>
     /// <returns>All ChildNodes.</returns>
-    protected HtmlNode[] QueryChildNodes(DefinitionsPack pack) => this.RootNode
+    protected IElement[] QueryChildNodes(DefinitionsPack pack) => this.RootNode
         .QuerySelectorAll(pack.Selector)
-        .Where(x => x.Name != "#text")
+        .Where(x => x.NodeName != "#text")
         .ToArray();
 
     /// <summary>
@@ -53,15 +52,15 @@ protected HtmlNode[] QueryChildNodes(DefinitionsPack pack) => this.RootNode
     /// <param name="pagedDefinition">Parser definition</param>
     /// <returns>List of nodes</returns>
     /// <exception cref="ArgumentException"></exception>
-    protected HtmlNode[] QueryContainer<TEntry>(PagedDefinition<TEntry> pagedDefinition) where TEntry : PagedEntryDefinition
+    protected IElement[] QueryContainer<TEntry>(PagedDefinition<TEntry> pagedDefinition) where TEntry : PagedEntryDefinition
     {
         var entryDef = pagedDefinition.Entry;
 
         if (entryDef == null)
             throw new ArgumentException("Could not get entry definition");
 
         return QueryNode(pagedDefinition.Root)
-            ?.QuerySelectorAll(entryDef.Root.Selector).ToArray() ?? Array.Empty<HtmlNode>();
+            ?.QuerySelectorAll(entryDef.Root.Selector).ToArray() ?? Array.Empty<IElement>();
     }
 
     /// <summary>
@@ -78,13 +77,12 @@ protected HtmlNode[] QueryContainer<TEntry>(PagedDefinition<TEntry> pagedDefinit
     /// <returns>InnerText of the node or empty string on parse error.</returns>
     protected string Parse(DefinitionsPack pack)
     {
-        if (!string.IsNullOrEmpty(pack.Regex))
-        {
-            var res = ParseRegex(pack);
+        if (string.IsNullOrEmpty(pack.Regex)) 
+            return ParseInnerText(pack);
+        var res = ParseRegex(pack);
 
-            if (res.Count != 0)
-                return res[1].Value;
-        }
+        if (res.Count != 0)
+            return res[1].Value;
 
         return ParseInnerText(pack);
     }
@@ -100,7 +98,7 @@ protected string ParseInnerText(DefinitionsPack pack, bool noAttribute = false)
         var node = QueryNode(pack);
 
         // Handle default attribute parsing
-        var text = !string.IsNullOrEmpty(pack.Attribute) && !noAttribute ? ParseAttribute(pack) : node?.InnerText;
+        var text = !string.IsNullOrEmpty(pack.Attribute) && !noAttribute ? ParseAttribute(pack) : node?.TextContent;
 
         return !string.IsNullOrEmpty(text) ? HttpUtility.HtmlDecode(text) : "";
     }
@@ -147,7 +145,7 @@ protected string ParseDirectInnerText(DefinitionsPack pack, bool noAttribute = f
 
         var text = !string.IsNullOrEmpty(pack.Attribute) && !noAttribute
             ? ParseAttribute(pack)
-            : node?.GetDirectInnerText();
+            : node?.ChildNodes.OfType<IText>().Select(m => m.Text).FirstOrDefault();
 
         return !string.IsNullOrEmpty(text) ? HttpUtility.HtmlDecode(text) : "";
     }

NetStone/Model/Parseables/CWLS/LodestoneCrossworldLinkshell.cs

@@ -1,5 +1,5 @@
 using System.Collections.Generic;
-using HtmlAgilityPack;
+using AngleSharp.Dom;
 using NetStone.Definitions;
 using NetStone.Definitions.Model.CWLS;
 using NetStone.Model.Parseables.CWLS.Members;
@@ -21,7 +21,7 @@ public class LodestoneCrossworldLinkshell : PaginatedIdResult<LodestoneCrossworl
     /// <param name="rootNode">The root document node of the page.</param>
     /// <param name="container">The <see cref="DefinitionsContainer"/> holding definitions to be used to access data.</param>
     /// <param name="id">The ID of the cross world linkshell.</param>
-    public LodestoneCrossworldLinkshell(LodestoneClient client, HtmlNode rootNode, DefinitionsContainer container, string id) 
+    public LodestoneCrossworldLinkshell(LodestoneClient client, IElement rootNode, DefinitionsContainer container, string id) 
         : base(rootNode,container.CrossworldLinkshellMember,client.GetCrossworldLinkshell,id)
     {
         this.definition = container.CrossworldLinkshell;

NetStone/Model/Parseables/CWLS/Members/CrossworldLinkshellMemberEntry.cs

@@ -1,4 +1,4 @@
-using HtmlAgilityPack;
+using AngleSharp.Dom;
 using NetStone.Definitions.Model.CWLS;
 
 namespace NetStone.Model.Parseables.CWLS.Members;
@@ -14,7 +14,7 @@ public class CrossworldLinkshellMemberEntry : LodestoneParseable
     /// </summary>
     /// <param name="rootNode">Root html node of this entry</param>
     /// <param name="definition">Css and regex definition</param>
-    public CrossworldLinkshellMemberEntry(HtmlNode rootNode, CrossworldLinkshellMemberEntryDefinition definition) : base(rootNode)
+    public CrossworldLinkshellMemberEntry(IElement rootNode, CrossworldLinkshellMemberEntryDefinition definition) : base(rootNode)
     {
         this.definition = definition;
     }

NetStone/Model/Parseables/Character/Achievement/CharacterAchievementEntry.cs

@@ -1,6 +1,6 @@
 using System;
 using System.Linq;
-using HtmlAgilityPack;
+using AngleSharp.Dom;
 using NetStone.Definitions.Model.Character;
 
 namespace NetStone.Model.Parseables.Character.Achievement;
@@ -17,7 +17,7 @@ public class CharacterAchievementEntry : LodestoneParseable
     /// </summary>
     /// <param name="rootNode">Root html node of this entry</param>
     /// <param name="definition">Css and regex definition</param>
-    public CharacterAchievementEntry(HtmlNode rootNode, CharacterAchievementEntryDefinition definition) : base(rootNode)
+    public CharacterAchievementEntry(IElement rootNode, CharacterAchievementEntryDefinition definition) : base(rootNode)
     {
         this.definition = definition;
     }

NetStone/Model/Parseables/Character/Achievement/CharacterAchievementPage.cs

@@ -1,5 +1,5 @@
 using System.Collections.Generic;
-using HtmlAgilityPack;
+using AngleSharp.Dom;
 using NetStone.Definitions.Model.Character;
 
 namespace NetStone.Model.Parseables.Character.Achievement;
@@ -18,7 +18,7 @@ public class CharacterAchievementPage : PaginatedIdResult<CharacterAchievementPa
     /// <param name="rootNode">Root node of the achievement page</param>
     /// <param name="definition">Parse definition pack</param>
     /// <param name="charId">ID of the character</param>
-    public CharacterAchievementPage(LodestoneClient client, HtmlNode rootNode, 
+    public CharacterAchievementPage(LodestoneClient client, IElement rootNode, 
                                     CharacterAchievementDefinition definition,string charId) 
         : base(rootNode, definition, client.GetCharacterAchievement, charId)
     {

NetStone/Model/Parseables/Character/CharacterAttributes.cs

@@ -1,5 +1,5 @@
 using System;
-using HtmlAgilityPack;
+using AngleSharp.Dom;
 using NetStone.Definitions.Model.Character;
 
 namespace NetStone.Model.Parseables.Character;
@@ -16,7 +16,7 @@ public class CharacterAttributes : LodestoneParseable
     /// </summary>
     /// <param name="rootNode">Root HTML node of the character profile page on Lodestone</param>
     /// <param name="definition">Definitions on how to parse attributes from the HTML</param>
-    public CharacterAttributes(HtmlNode rootNode, CharacterAttributesDefinition definition) : base(rootNode)
+    public CharacterAttributes(IElement rootNode, CharacterAttributesDefinition definition) : base(rootNode)
     {
         this.definition = definition;
     }

NetStone/Model/Parseables/Character/ClassJob/CharacterClassJob.cs

@@ -1,5 +1,5 @@
 using System.Collections.Generic;
-using HtmlAgilityPack;
+using AngleSharp.Dom;
 using NetStone.Definitions.Model.Character;
 
 namespace NetStone.Model.Parseables.Character.ClassJob;
@@ -16,7 +16,7 @@ public class CharacterClassJob : LodestoneParseable
     /// </summary>
     /// <param name="rootNode">Root html node of Lodestone page</param>
     /// <param name="definition">Definition to parse ClassJobs</param>
-    public CharacterClassJob(HtmlNode rootNode, CharacterClassJobDefinition definition) : base(rootNode)
+    public CharacterClassJob(IElement rootNode, CharacterClassJobDefinition definition) : base(rootNode)
     {
         this.definition = definition;
     }