问题:
- 非可见标签（script、style）
- 块级标签
- 行内标签
- 换行符
- 可换行空格（前导、尾随和多个空格）
- 硬空格
- 实体
代数决策：
plain-text = Process(Plain(html))
Plain(node-s) => Plain(node-0), Plain(node-1), ..., Plain(node-N)
Plain(BR) => BR
Plain(not-visible-element(child-s)) => nil
Plain(block-element(child-s)) => BS, Plain(child-s), BE
Plain(inline-element(child-s)) => Plain(child-s)
Plain(text) => ch-0, ch-1, .., ch-N
Process(symbol-s) => Process(start-line, symbol-s)
Process(start-line, BR, symbol-s) => Print('\n'), Process(start-line, symbol-s)
Process(start-line, BS, symbol-s) => Process(start-line, symbol-s)
Process(start-line, BE, symbol-s) => Process(start-line, symbol-s)
Process(start-line, hard-space, symbol-s) => Print(' '), Process(not-ws, symbol-s)
Process(start-line, space, symbol-s) => Process(start-line, symbol-s)
Process(start-line, common-symbol, symbol-s) => Print(common-symbol),
Process(not-ws, symbol-s)
Process(not-ws, BR|BS|BE, symbol-s) => Print('\n'), Process(start-line, symbol-s)
Process(not-ws, hard-space, symbol-s) => Print(' '), Process(not-ws, symbol-s)
Process(not-ws, space, symbol-s) => Process(ws, symbol-s)
Process(not-ws, common-symbol, symbol-s) => Process(ws, symbol-s)
Process(ws, BR|BS|BE, symbol-s) => Print('\n'), Process(start-line, symbol-s)
Process(ws, hard-space, symbol-s) => Print(' '), Print(' '),
Process(not-ws, symbol-s)
Process(ws, space, symbol-s) => Process(ws, symbol-s)
Process(ws, common-symbol, symbol-s) => Print(' '), Print(common-symbol),
Process(not-ws, symbol-s)
在 HtmlAgilityPack 和 System.Xml.Linq 之间做 C# 决策:
public static string ToPlainText(this HtmlAgilityPack.HtmlDocument doc)
{
var builder = new System.Text.StringBuilder();
var state = ToPlainTextState.StartLine;
Plain(builder, ref state, new[]{doc.DocumentNode});
return builder.ToString();
}
static void Plain(StringBuilder builder, ref ToPlainTextState state, IEnumerable<HtmlAgilityPack.HtmlNode> nodes)
{
foreach (var node in nodes)
{
if (node is HtmlAgilityPack.HtmlTextNode)
{
var text = (HtmlAgilityPack.HtmlTextNode)node;
Process(builder, ref state, HtmlAgilityPack.HtmlEntity.DeEntitize(text.Text).ToCharArray());
}
else
{
var tag = node.Name.ToLower();
if (tag == "br")
{
builder.AppendLine();
state = ToPlainTextState.StartLine;
}
else if (NonVisibleTags.Contains(tag))
{
}
else if (InlineTags.Contains(tag))
{
Plain(builder, ref state, node.ChildNodes);
}
else
{
if (state != ToPlainTextState.StartLine)
{
builder.AppendLine();
state = ToPlainTextState.StartLine;
}
Plain(builder, ref state, node.ChildNodes);
if (state != ToPlainTextState.StartLine)
{
builder.AppendLine();
state = ToPlainTextState.StartLine;
}
}
}
}
}
public static string ToPlainText(this IEnumerable<XNode> nodes)
{
var builder = new System.Text.StringBuilder();
var state = ToPlainTextState.StartLine;
Plain(builder, ref state, nodes);
return builder.ToString();
}
static void Plain(StringBuilder builder, ref ToPlainTextState state, IEnumerable<XNode> nodes)
{
foreach (var node in nodes)
{
if (node is XElement)
{
var element = (XElement)node;
var tag = element.Name.LocalName.ToLower();
if (tag == "br")
{
builder.AppendLine();
state = ToPlainTextState.StartLine;
}
else if (NonVisibleTags.Contains(tag))
{
}
else if (InlineTags.Contains(tag))
{
Plain(builder, ref state, element.Nodes());
}
else
{
if (state != ToPlainTextState.StartLine)
{
builder.AppendLine();
state = ToPlainTextState.StartLine;
}
Plain(builder, ref state, element.Nodes());
if (state != ToPlainTextState.StartLine)
{
builder.AppendLine();
state = ToPlainTextState.StartLine;
}
}
}
else if (node is XText)
{
var text = (XText)node;
Process(builder, ref state, text.Value.ToCharArray());
}
}
}
public static void Process(System.Text.StringBuilder builder, ref ToPlainTextState state, params char[] chars)
{
foreach (var ch in chars)
{
if (char.IsWhiteSpace(ch))
{
if (IsHardSpace(ch))
{
if (state == ToPlainTextState.WhiteSpace)
builder.Append(' ');
builder.Append(' ');
state = ToPlainTextState.NotWhiteSpace;
}
else
{
if (state == ToPlainTextState.NotWhiteSpace)
state = ToPlainTextState.WhiteSpace;
}
}
else
{
if (state == ToPlainTextState.WhiteSpace)
builder.Append(' ');
builder.Append(ch);
state = ToPlainTextState.NotWhiteSpace;
}
}
}
static bool IsHardSpace(char ch)
{
return ch == 0xA0 || ch == 0x2007 || ch == 0x202F;
}
private static readonly HashSet<string> InlineTags = new HashSet<string>
{
"b", "big", "i", "small", "tt", "abbr", "acronym",
"cite", "code", "dfn", "em", "kbd", "strong", "samp",
"var", "a", "bdo", "br", "img", "map", "object", "q",
"script", "span", "sub", "sup", "button", "input", "label",
"select", "textarea"
};
private static readonly HashSet<string> NonVisibleTags = new HashSet<string>
{
"script", "style"
};
public enum ToPlainTextState
{
StartLine = 0,
NotWhiteSpace,
WhiteSpace,
}
}
例子：
// <div> 1 </div> 2 <div> 3 </div>
1
2
3
// <div>1 <br/><br/>  <b> 2 </b> <div> </div><div> </div>  3</div>
1
2
3
// <span>1<style> text </style><i>2</i></span>3
123
//<div>
// <div>
// <div>
// line1
// </div>
// </div>
//</div>
//<div>line2</div>
line1
line2
