Module:Wikitext Parsing

require("strict")--Helper functionslocal function startswith(text, subtext)return string.sub(text, 1, #subtext) == subtextendlocal function endswith(text, subtext)return string.sub(text, -#subtext, -1) == subtextendlocal function allcases(s)return s:gsub("%a", function(c) return "["..c:upper()..c:lower().."]"end)endlocal trimcache = {}local whitespace = {[" "]=1, ["\n"]=1, ["\t"]=1, ["\r"]=1}local function cheaptrim(str) --mw.text.trim is surprisingly expensive, so here's an alternative approachlocal quick = trimcache[str]if quick thenreturn quickelse-- local out = string.gsub(str, "^%s*(.-)%s*$", "%1")local lowEndfor i = 1,#str doif not whitespace[string.sub(str, i, i)] thenlowEnd = ibreakendendif not lowEnd thentrimcache[str] = ""return ""endfor i = #str,1,-1 doif not whitespace[string.sub(str, i, i)] thenlocal out = string.sub(str, lowEnd, i)trimcache[str] = outreturn outendendendend--[=[ Implementation notes---- NORMAL HTML TAGS ----Tags are very strict on how they want to start, but loose on how they end.The start must strictly follow <[tAgNaMe](%s|>) with no room for whitespace inthe tag's name, but may then flow as they want afterwards, making<div\nclass\n=\n"\nerror\n"\n> validThere's no sense of escaping < or >E.g. <div class="error\>"> will end at \> despite it being inside a quote <div class="<span class="error">error</span>"> will not process the larger divIf a tag has no end, it will consume all text instead of not processing---- NOPROCESSING TAGS (nowiki, pre, syntaxhighlight, source, etc.) ----(In most comments, <source> will not be mentioned. This is because it is thedeprecated version of <syntaxhighlight>)No-Processing tags have some interesting differences to the above rules.For example, their syntax is a lot stricter. While an opening tag appears tofollow the same set of rules, A closing tag can't have any sort of extraformatting period. While </div a/a> is valid, </nowiki a/a> isn't - onlynewlines and spaces/tabs are allowed in closing tags.Note that, even though <pre> tags cause a visual change when the ending tag hasextra formatting, it won't cause the no-processing effects. For some reason, theformat must be strict for that to apply.Both the content inside the tag pair and the content inside each side of thepair is not processed. E.g. <nowiki |}}>|}}</nowiki> would have both of the |}}escaped in practice.When something in the code is referenced to as a "Nowiki Tag", it means a tagwhich causes wiki text to not be processed, which includes <nowiki>, <pre>,and <syntaxhighlight>Since we only care about these tags, we can ignore the idea of an interceptingtag preventing processing, and just go straight for the first ending we can findIf there is no ending to find, the tag will NOT consume the rest of the text interms of processing behaviour (though <pre> will appear to have an effect).Even if there is no end of the tag, the content inside the opening half willstill be unprocessed, meaning {{X20|<nowiki }}>}} wouldn't end at the first }}despite there being no ending to the tag.Note that there are some tags, like <math>, which also function like <nowiki>which are included in this aswell. Some other tags, like <ref>, have far toounpredictable behaviour to be handled currently (they'd have to be split andprocessed as something seperate - its complicated, but maybe not impossible.)I suspect that every tag listed in [[Special:Version]] may behave somewhat likethis, but that's far too many cases worth checking for rarely used tags that maynot even have a good reason to contain {{ or }} anyways, so we leave them alone.---- HTML COMMENTS AND INCLUDEONLY ----HTML Comments are about as basic as it could get for thisStart at <!--, end at -->, no extra conditions. Simple enoughIf a comment has no end, it will eat all text instead of not being processedincludeonly tags function mostly like a regular nowiki tag, with the exceptionthat the tag will actually consume all future text if not given an ending asopposed to simply giving up and not changing anything. Due to complications andthe fact that this is far less likely to be present on a page, aswell as beingsomething that may not want to be escaped, includeonly tags are ignored duringour processing--]=]local validtags = {nowiki=1, pre=1, syntaxhighlight=1, source=1, math=1}--This function expects the string to start with the taglocal function TestForNowikiTag(text, scanPosition)local tagName = (string.match(text, "^<([^\n />]+)", scanPosition) or ""):lower()if not validtags[tagName] thenreturn nilendlocal nextOpener = string.find(text, "<", scanPosition+1) or -1local nextCloser = string.find(text, ">", scanPosition+1) or -1if nextCloser > -1 and (nextOpener == -1 or nextCloser < nextOpener) thenlocal startingTag = string.sub(text, scanPosition, nextCloser)--We have our starting tag (E.g. '<pre style="color:red">')--Now find our ending...if endswith(startingTag, "/>") then --self-closing tag (we are our own ending)return {Tag = tagName,Start = startingTag,Content = "", End = "",Length = #startingTag}elselocal endingTagStart, endingTagEnd = string.find(text, "</"..allcases(tagName).."[ \t\n]*>", scanPosition)if endingTagStart then --Regular tag formationlocal endingTag = string.sub(text, endingTagStart, endingTagEnd)local tagContent = string.sub(text, nextCloser+1, endingTagStart-1)return {Tag = tagName,Start = startingTag,Content = tagContent,End = endingTag,Length = #startingTag + #tagContent + #endingTag}else --Content inside still needs escaping (also linter error!)return {Tag = tagName,Start = startingTag,Content = "", End = "",Length = #startingTag}endendendreturn nilendlocal function TestForComment(text, scanPosition) --Like TestForNowikiTag but for <!-- -->if string.match(text, "^<!%-%-", scanPosition) thenlocal commentEnd = string.find(text, "-->", scanPosition+4, true)if commentEnd thenreturn {Start = "<!--", End = "-->",Content = string.sub(text, scanPosition+4, commentEnd-1),Length = commentEnd-scanPosition+3}else --Consumes all text if not given an endingreturn {Start = "<!--", End = "",Content = string.sub(text, scanPosition+4),Length = #text-scanPosition+1}endendreturn nilend--[[ Implementation notesThe goal of this function is to escape all text that wouldn't be parsed if itwas preprocessed (see above implementation notes).Using keepComments will keep all HTML comments instead of removing them. Theywill still be escaped regardless to avoid processing errors--]]local function PrepareText(text, keepComments)local newtext = {}local scanPosition = 1while true dolocal NextCheck = string.find(text, "<[NnSsPpMm!]", scanPosition) --Advance to the next potential tag we care aboutif not NextCheck then --Donenewtext[#newtext+1] =  string.sub(text,scanPosition)breakendnewtext[#newtext+1] = string.sub(text,scanPosition,NextCheck-1)scanPosition = NextChecklocal Comment = TestForComment(text, scanPosition)if Comment thenif keepComments thennewtext[#newtext+1] = Comment.Start .. mw.text.nowiki(Comment.Content) .. Comment.EndendscanPosition = scanPosition + Comment.Lengthelselocal Tag = TestForNowikiTag(text, scanPosition)if Tag thenlocal newTagStart = "<" .. mw.text.nowiki(string.sub(Tag.Start,2,-2)) .. ">"local newTagEnd = Tag.End == "" and "" or --Respect no tag ending"</" .. mw.text.nowiki(string.sub(Tag.End,3,-2)) .. ">"local newContent = mw.text.nowiki(Tag.Content)newtext[#newtext+1] = newTagStart .. newContent .. newTagEndscanPosition = scanPosition + Tag.Lengthelse --Nothing special, move on...newtext[#newtext+1] = string.sub(text, scanPosition, scanPosition)scanPosition = scanPosition + 1endendendreturn table.concat(newtext, "")end--[=[ Implementation notesThis function is an alternative to Transcluder's getParameters which considersthe potential for a singular { or } or other odd syntax that %b doesn't like tobe in a parameter's value.When handling the difference between {{ and {{{, mediawiki will attempt to matchas many sequences of {{{ as possible before matching a {{E.g. {{{{A}}}} -> { {{{A}}} } {{{{{{{{Text|A}}}}}}}} -> {{ {{{ {{{Text|A}}} }}} }}If there aren't enough triple braces on both sides, the parser will compromisefor a template interpretation.E.g. {{{{A}} }} -> {{ {{ A }} }}While there are technically concerns about things such as wikilinks breakingtemplate processing (E.g. {{[[}}]]}} doesn't stop at the first }}), it shouldn'tbe our job to process inputs perfectly when the input has garbage ({ / } isn'tlegal in titles anyways, so if something's unmatched in a wikilink, it'sguaranteed GIGO)Setting dontEscape will prevent running the input text through EET. Avoidsetting this to true if you don't have to set it.Returned values:A table of all templates. Template data goes as follows: Text: The raw text of the template Name: The name of the template Args: A list of arguments Children: A list of immediate template children--]=]--Helper functionslocal function boundlen(pair)return pair.End-pair.Start+1end--Main functionlocal function ParseTemplates(InputText, dontEscape)--Setupif not dontEscape thenInputText = PrepareText(InputText)endlocal function finalise(text)if not dontEscape thenreturn mw.text.decode(text)elsereturn textendendlocal function CreateContainerObj(Container)Container.Text = {}Container.Args = {}Container.ArgOrder = {}Container.Children = {}-- Container.Name = nil-- Container.Value = nil-- Container.Key = nilContainer.BeyondStart = falseContainer.LastIndex = 1Container.finalise = finalisefunction Container:HandleArgInput(character, internalcall)if not internalcall thenself.Text[#self.Text+1] = characterendif character == "=" thenif self.Key thenself.Value[#self.Value+1] = characterelseself.Key = cheaptrim(self.Value and table.concat(self.Value, "") or "")self.Value = {}endelse --"|" or "}"if not self.Name thenself.Name = cheaptrim(self.Value and table.concat(self.Value, "") or "")self.Value = nilelseself.Value = self.finalise(self.Value and table.concat(self.Value, "") or "")if self.Key thenself.Key = self.finalise(self.Key)self.Args[self.Key] = cheaptrim(self.Value)self.ArgOrder[#self.ArgOrder+1] = self.Keyelselocal Key = tostring(self.LastIndex)self.Args[Key] = self.Valueself.ArgOrder[#self.ArgOrder+1] = Keyself.LastIndex = self.LastIndex + 1endself.Key = nilself.Value = nilendendendfunction Container:AppendText(text, ftext)self.Text[#self.Text+1] = (ftext or text)if not self.Value thenself.Value = {}endself.BeyondStart = self.BeyondStart or (#table.concat(self.Text, "") > 2)if self.BeyondStart thenself.Value[#self.Value+1] = textendendfunction Container:Clean(IsTemplate)self.Text = table.concat(self.Text, "")if self.Value and IsTemplate thenself.Value = {string.sub(table.concat(self.Value, ""), 1, -3)} --Trim ending }}self:HandleArgInput("|", true) --Simulate endingendself.Value = nilself.Key = nilself.BeyondStart = nilself.LastIndex = nilself.finalise = nilself.HandleArgInput = nilself.AppendText = nilself.Clean = nilendreturn Containerend--Step 1: Find and escape the content of all wikilinks on the page, which are stronger than templates (see implementation notes)local scannerPosition = 1local wikilinks = {}local openWikilinks = {}while true dolocal Position, _, Character = string.find(InputText, "([%[%]])%1", scannerPosition)if not Position then --DonebreakendscannerPosition = Position+2 --+2 to pass the [[ / ]]if Character == "[" then --Add a [[ to the pending wikilink queueopenWikilinks[#openWikilinks+1] = Positionelse --Pair up the ]] to any available [[if #openWikilinks >= 1 thenlocal start = table.remove(openWikilinks) --Pop the latest [[wikilinks[start] = {Start=start, End=Position+1, Type="Wikilink"} --Note the pairendendend--Step 2: Find the bounds of every valid template and variable ({{ and {{{)local scannerPosition = 1local templates = {}local variables = {}local openBrackets = {}while true dolocal Start, _, Character = string.find(InputText, "([{}])%1", scannerPosition)if not Start then --Done (both 9e9)breakendlocal _, End = string.find(InputText, "^"..Character.."+", Start)scannerPosition = Start --Get to the {{ / }} setif Character == "{" then --Add the {{+ set to the queueopenBrackets[#openBrackets+1] = {Start=Start, End=End}else --Pair up the }} to any available {{, accounting for {{{ / }}}local BracketCount = End-Start+1while BracketCount >= 2 and #openBrackets >= 1 dolocal OpenSet = table.remove(openBrackets)if boundlen(OpenSet) >= 3 and BracketCount >= 3 then --We have a {{{variable}}} (both sides have 3 spare)variables[OpenSet.End-2] = {Start=OpenSet.End-2, End=scannerPosition+2, Type="Variable"} --Done like this to ensure chronological orderBracketCount = BracketCount - 3OpenSet.End = OpenSet.End - 3scannerPosition = scannerPosition + 3else --We have a {{template}} (both sides have 2 spare, but at least one side doesn't have 3 spare)templates[OpenSet.End-1] = {Start=OpenSet.End-1, End=scannerPosition+1, Type="Template"} --Done like this to ensure chronological orderBracketCount = BracketCount - 2OpenSet.End = OpenSet.End - 2scannerPosition = scannerPosition + 2endif boundlen(OpenSet) >= 2 then --Still has enough data left, leave it inopenBrackets[#openBrackets+1] = OpenSetendendendscannerPosition = End --Now move past the bracket setend--Step 3: Re-trace every object using their known bounds, collecting our parameters with (slight) easelocal scannerPosition = 1local activeObjects = {}local finalObjects = {}while true dolocal LatestObject = activeObjects[#activeObjects] --Commonly needed objectlocal NNC, _, Character --NNC = NextNotableCharacterif LatestObject thenNNC, _, Character = string.find(InputText, "([{}%[%]|=])", scannerPosition)elseNNC, _, Character = string.find(InputText, "([{}])", scannerPosition) --We are only after templates right nowendif not NNC thenbreakendif NNC > scannerPosition and LatestObject thenlocal scannedContent = string.sub(InputText, scannerPosition, NNC-1)LatestObject:AppendText(scannedContent, finalise(scannedContent))endscannerPosition = NNC+1if Character == "{" or Character == "[" thenlocal Container = templates[NNC] or variables[NNC] or wikilinks[NNC]if Container thenCreateContainerObj(Container)if Container.Type == "Template" thenContainer:AppendText("{{")scannerPosition = NNC+2elseif Container.Type == "Variable" thenContainer:AppendText("{{{")scannerPosition = NNC+3else --WikilinkContainer:AppendText("[[")scannerPosition = NNC+2endif LatestObject and Container.Type == "Template" then --Only templates count as childrenLatestObject.Children[#LatestObject.Children+1] = ContainerendactiveObjects[#activeObjects+1] = Containerelseif LatestObject thenLatestObject:AppendText(Character)endelseif Character == "}" or Character == "]" thenif LatestObject thenLatestObject:AppendText(Character)if LatestObject.End == NNC thenif LatestObject.Type == "Template" thenLatestObject:Clean(true)finalObjects[#finalObjects+1] = LatestObjectelseLatestObject:Clean(false)endactiveObjects[#activeObjects] = nillocal NewLatest = activeObjects[#activeObjects]if NewLatest thenNewLatest:AppendText(LatestObject.Text) --Append to new latestendendendelse --| or =if LatestObject thenLatestObject:HandleArgInput(Character)endendend--Step 4: Fix the orderlocal FixedOrder = {}local SortableReference = {}for _,Object in next,finalObjects doSortableReference[#SortableReference+1] = Object.Startendtable.sort(SortableReference)for i = 1,#SortableReference dolocal start = SortableReference[i]for n,Object in next,finalObjects doif Object.Start == start thenfinalObjects[n] = nilObject.Start = nil --Final cleanupObject.End = nilObject.Type = nilFixedOrder[#FixedOrder+1] = Objectbreakendendend--Finished, returnreturn FixedOrderendlocal p = {}--Main entry pointsp.PrepareText = PrepareTextp.ParseTemplates = ParseTemplates--Extra entry points, not really requiredp.TestForNowikiTag = TestForNowikiTagp.TestForComment = TestForCommentreturn p--[==[ console testslocal s = [=[Hey!{{Text|<nowiki | ||>Hey! }}A</nowiki>|<!--AAAAA|AAA-->Should see|Shouldn't see}}]=]local out = p.PrepareText(s)mw.logObject(out)local s = [=[B<!--Hey!-->A]=]local out = p.TestForComment(s, 2)mw.logObject(out); mw.log(string.sub(s, 2, out.Length))local a = p.ParseTemplates([=[{{User:Aidan9382/templates/dummy|A|B|C {{{A|B}}} { } } {|<nowiki>D</nowiki>|<pre>E|F</pre>|G|=|a=|A  =  [[{{PAGENAME}}|A=B]]{{Text|1==<nowiki>}}</nowiki>}}|A B=Success}}]=])mw.logObject(a)]==]