experimental: add the ability to drop inputs and lexer-level alternates

1 files changed, 34 insertions, 10 deletions

diff --git a/patok.lua b/patok.lua
index 55cc8dc..d0937d5 100644
--- a/patok.lua
+++ b/patok.lua
@@ -1,8 +1,28 @@
+local id = function(in) return function() return in end end
+local pa = function(pt) return function(in) return in:find(pt) end
+
 local meta = {}
 function meta:__call(p)
 	if p == nil then return self.out end
 	for k, v in pairs(p) do
-		table.insert(self.out.tokens, {name=k, pattern='^' .. v})
+		-- we accept: string | { patterns..., [drop = true|false|string|func] }
+		-- we output: { '^' .. patterns, drop = func, name = string }
+		if type(v) == 'string' then
+			v = { v }
+		end
+		-- assume table
+		if v.drop == nil then
+			v.drop = id(false)
+		elseif type(v.drop) == 'string' then
+			v.drop = pa(v.drop)
+		elseif type(v.drop) == 'boolean' then
+			v.drop = id(v.drop)
+		end
+		v.name = k
+
+		for ii in ipairs(v) do v[ii] = '^' .. v[ii] end
+
+		table.insert(self.out.tokens, v)
 	end
 	return self
 end
@@ -16,15 +36,19 @@ function new (p)
 		end,
 		next   = function (self)
 			for _, v in ipairs(self.tokens) do
-				local out = {string.find(self.input, v.pattern, self.index)}
-				if out[1] == self.index and out[2] >= out[1] then
-					self.index = out[2]+1
-					return {
-						type  = v.name,
-						start = out[1],
-						stop  = out[2],
-						value = string.sub(self.input, out[1], out[2])
-					}
+				for _, vv in ipairs(v) do
+					local out = {string.find(self.input, vv, self.index)}
+					if out[1] == self.index and out[2] >= out[1] then
+						self.index = out[2]+1
+						local match = {
+							type  = v.name,
+							start = out[1],
+							stop  = out[2],
+							value = string.sub(self.input, out[1], out[2])
+						}
+						if v.drop(match.value) then return self.next() end -- skip this one
+						return match
+					end
 				end
 			end
 			-- we didn't find anything