Module:UtilsSchema

local p = {} local h = {}

local utilsValidate = require("Module:UtilsArg") local utilsError = require("Module:UtilsError") local utilsString = require("Module:UtilsString") local utilsTable = require("Module:UtilsTable")

local SYMBOLS = { optional = "[%s]", required = "%s!", default = "%s=%s", array = "{%s}", map = "<%s, %s>", oneOf = "%s | %s", allOf = "%s & %s", combinationGroup = "(%s)", }

p.TYPES = { oneOf = "oneOf", array = "array", record = "record", map = "map" }

function p.validate(schemas, schemaName, data, dataName) -- first validate the schema itself h.collectReferences(p.Schemas["Schema"]) local schemaErrors = h.validate(p.Schemas["Schema"], schemas[schemaName], schemaName) if #schemaErrors > 0 then utilsError.warn(string.format("Schema  is invalid.", schemaName)) end -- then validate the data h.collectReferences(schemas[schemaName]) local errors = h.validate(schemas[schemaName], data, dataName) if #errors > 0 then utilsError.warn(string.format(" is invalid according to schema  ", dataName, schemaName)) return "Category:Modules with invalid data", errors end end

function p.getTypeDefinitions(schemas, schemaName, formattingFn) local schema = schemas[schemaName] h.collectReferences(schema) h.minRefDepths(schema) return h.getTypeDefs(schemaName, schema, formattingFn) end

function h.collectReferences(schema) h.references = {} h.references["#"] = schema for k, v in pairs(schema.definitions or {}) do h.references["#/definitions/" .. k] = v		h.collectIdReferences(v) end h.collectIdReferences(schema) end function h.collectIdReferences(schema) h.walkSchema(function(schemaNode)		if schemaNode._id then			h.references[schemaNode._id] = schemaNode		end	end, schema) end

function h.resolveReference(schemaNode) if schemaNode._ref then local referenceNode = h.references[schemaNode._ref] if not referenceNode then mw.addWarning(string.format("%s not found", code(mw.text.nowiki(schemaNode._ref)))) else local resolvedSchema = utilsTable.merge({}, h.references[schemaNode._ref], schemaNode) schemaNode = utilsTable.merge({}, schemaNode, resolvedSchema) schemaNode._ref = nil end end return schemaNode end

function h.walkSchema(fn, schemaNode, path) path = path or {} local continue = fn(schemaNode, path) if continue == false then return end if schemaNode.items then h.walkSchema(fn, schemaNode.items, utilsTable.concat(path, "items")) end if schemaNode.keys then h.walkSchema(fn, schemaNode.keys, utilsTable.concat(path, "keys")) end if schemaNode.values then h.walkSchema(fn, schemaNode.values, utilsTable.concat(path, "values")) end if schemaNode.properties then for i, v in ipairs(schemaNode.properties) do			local keyPath = utilsTable.concat(path, "properties", v.name) h.walkSchema(fn, v, keyPath) end end if schemaNode.oneOf then for i, v in ipairs(schemaNode.oneOf) do			h.walkSchema(fn, v, utilsTable.concat(path, "oneOf")) end end if schemaNode.allOf then for i, v in ipairs(schemaNode.allOf) do			h.walkSchema(fn, v, utilsTable.concat(path, "allOf")) end end if schemaNode.definitions then for k, v in pairs(schemaNode.definitions) do			h.walkSchema(fn, v, utilsTable.concat(path, "definitions", k)) end end end

-- This is to ensure that the documentation for recursive refs is shown only once. function h.minRefDepths(schema) h.minDepthNode = {} local minDepths = {} h.walkSchema(function(schemaNode, path)		if schemaNode._ref then			minDepths[schemaNode._ref] = math.min(minDepths[schemaNode._ref] or 9000, #path)			if #path == minDepths[schemaNode._ref] then				h.minDepthNode[schemaNode._ref] = schemaNode			end		end	end, schema) end function h.hasRefs(schema) local hasRefs = false h.walkSchema(function(schemaNode)		if schemaNode._ref then			hasRefs = true			return false		end	end, schema) return hasRefs end function h.showSubkeys(schemaNode) return not schemaNode._ref or h.minDepthNode[schemaNode._ref] == schemaNode or not h.hasRefs(h.references[schemaNode._ref]) end

function h.getTypeDefs(schemaName, schema, formattingFn, parentSchema, isSubschema) local typeLabel local showSubkeys = true if schema._ref and utilsString.startsWith("#/definitions", schema._ref) then showSubkeys = h.showSubkeys(schema) typeLabel = string.gsub(schema._ref, "#/definitions/", "") end schema = h.resolveReference(schema) local rawType = schema.type local symbolicType local subkeys if showSubkeys then if schema.type == p.TYPES.record then for _, prop in ipairs(schema.properties) do				subkeys = subkeys or {} local propDef = h.getTypeDefs(prop.name, prop, formattingFn, schema) table.insert(subkeys, propDef) end end if schema.type == p.TYPES.array then local subtypeKeys, subtype = h.getTypeDefs(nil, schema.items, formattingFn, schema) if #subtypeKeys > 0 then subkeys = subtypeKeys end symbolicType = string.format(SYMBOLS.array, subtype) end if schema.type == p.TYPES.map then local _, keyType = h.getTypeDefs(nil, schema.keys, formattingFn, schema) local valueDef, valueType = h.getTypeDefs(nil, schema.values, formattingFn, schema) symbolicType = string.format(SYMBOLS.map, keyType, valueType) subkeys = valueDef end if schema.oneOf then subkeys = subkeys or {} subkeys.oneOf = subkeys.oneOf or {} local subtypes = {} for i, subschema in ipairs(schema.oneOf) do				local keys, subtype = h.getTypeDefs(nil, subschema, formattingFn, schema, true) if string.find(subtype, "|") or string.find(subtype, "&") then subtype = string.format(SYMBOLS.combinationGroup, subtype) end table.insert(subtypes, subtype) subkeys.oneOf[i] = keys end symbolicType = subtypes[1] for i, subtype in ipairs(utilsTable.tail(subtypes)) do				symbolicType = string.format(SYMBOLS.oneOf, symbolicType, subtype) end end if schema.allOf then subkeys = subkeys or {} subkeys.allOf = {} local subtypes = {} for i, subschema in ipairs(schema.allOf) do				local keys, subtype = h.getTypeDefs(nil, subschema, formattingFn, schema) if string.find(subtype, "|") or string.find(subtype, "&") then subtype = string.format(SYMBOLS.combinationGroup, subtype) end table.insert(subtypes, subtype) subkeys.allOf[i] = keys end subtypes = utilsTable.unique(subtypes) symbolicType = subtypes[1] for i, subtype in ipairs(utilsTable.tail(subtypes)) do				symbolicType = string.format(SYMBOLS.allOf, symbolicType, subtype) end end end symbolicType = symbolicType or rawType or typeLabel local parentType = parentSchema and parentSchema.type local key = schemaName if schema.default then key = key and string.format(SYMBOLS.default, schemaName, schema.default) end if parentSchema and not parentSchema.allOf and not parentSchema.oneOf and parentType ~= p.TYPES.array and parentType ~= p.TYPES.map then -- otherwise leads to nonsense like [[string], instead of [{string}|string]		if schema.required then			symbolicType = string.format(SYMBOLS.required, symbolicType)		else			symbolicType = string.format(SYMBOLS.optional, symbolicType)			key = key and string.format(SYMBOLS.optional, key)		end	end	local formattedDef = formattingFn({		key = key,		subkeys = subkeys,		rawType = rawType,		typeLabel = typeLabel,		symbolicType = symbolicType,		desc = schema.desc,		parentType = parentType,		isSubschema = isSubschema,	})	return formattedDef, symbolicType end

function h.validate(schemaNode, data, dataName, dataPath, quiet, isKey) dataPath = dataPath or {} local value = utilsTable.property(dataPath)(data) local errPath = dataName .. utilsTable.path(dataPath) schemaNode = h.resolveReference(schemaNode) local errors = h.validatePrimitive(schemaNode, value, dataName, dataPath, quiet, isKey) if #errors > 0 then return errors end if schemaNode.allOf and value then local subschemaErrors = h.validateSubschemas(schemaNode.allOf, data, dataName, dataPath) local invalidSubschemas = utilsTable.keys(subschemaErrors) if #invalidSubschemas > 0 then invalidSubschemas = utilsTable.map(code)(invalidSubschemas) invalidSubschemas = mw.text.listToText(invalidSubschemas) local msg = string.format("%s does not match  sub-schemas %s", code(errPath), invalidSubschemas) if not quiet then utilsError.warn(msg, false) h.logSubschemaErrors(subschemaErrors) end table.insert(errors, {				path = errPath,				msg = msg,				errors = subschemaErrors			}) end end if schemaNode.oneOf and value then local subschemaErrors, validSubschemas = h.validateSubschemas(schemaNode.oneOf, data, dataName, dataPath) local invalidSubschemas = utilsTable.keys(subschemaErrors) if #validSubschemas == 0 then local msg = string.format("%s does not match any  sub-schemas.", code(errPath)) if not quiet then utilsError.warn(msg, false) h.logSubschemaErrors(subschemaErrors) end table.insert(errors, {				path = errPath,				msg = msg,				errors = subschemaErrors			}) end if #validSubschemas > 1 then validSubschemas = utilsTable.map(code)(validSubschemas) validSubschemas = mw.text.listToText(validSubschemas) local msg = string.format("%s matches  sub-schemas %s, but must match only one.", code(errPath), validSubschemas) if not quiet then utilsError.warn(msg, false) end table.insert(errors, {				path = errPath,				msg = msg,			}) end end if schemaNode.properties and value then for _, propSchema in pairs(schemaNode.properties) do			local keyPath = utilsTable.concat(dataPath, propSchema.name) errors = utilsTable.concat(errors, h.validate(propSchema, data, dataName, keyPath, quiet)) end if not schemaNode.additionalProperties then local schemaProps = utilsTable.map("name")(schemaNode.properties) local dataProps = utilsTable.keys(value) local undefinedProps = utilsTable.difference(dataProps)(schemaProps) if #undefinedProps > 0 then undefinedProps = mw.text.listToText(utilsTable.map(code)(undefinedProps)) local msg = string.format("Record %s has undefined properties: %s", code(errPath), undefinedProps) if not quiet then utilsError.warn(msg, false) end table.insert(errors, {					path = errPath,					msg = msg,				}) end end end if schemaNode.items and value then for i, item in ipairs(value) do			local itemPath = utilsTable.concat(dataPath, i)			errors = utilsTable.concat(errors, h.validate(schemaNode.items, data, dataName, itemPath, quiet)) end end if schemaNode.keys and schemaNode.values and value then for k, v in pairs(value) do			local keyPath = utilsTable.concat(dataPath, k)			errors = utilsTable.concat(errors, h.validatePrimitive(schemaNode.keys, k, dataName, dataPath, quiet, true)) errors = utilsTable.concat(errors, h.validate(schemaNode.values, data, dataName, keyPath, quiet)) end end return errors end

function h.validateSubschemas(subschemas, data, dataName, dataPath) local errors = {} local valids = {} for i, subschema in ipairs(subschemas) do		local key = subschema._ref and string.gsub(subschema._ref, "#/definitions/", "") or i		local err = h.validate(subschema, data, dataName, dataPath, true) if #err > 0 then errors[key] = err else table.insert(valids, key) end end return errors, valids end function h.logSubschemaErrors(subschemaErrors) for schemaKey, schemaErrors in pairs(subschemaErrors) do		for _, err in pairs(utilsTable.flatten(subschemaErrors)) do utilsError.warn(code(schemaKey) .. ": " .. err.msg, false) end end end

function h.validatePrimitive(schemaNode, value, dataName, dataPath, quiet, isKey) local errors = {} local addIfError = h.errorCollector(errors, value, dataName, dataPath, quiet, isKey) local validatorOptions = { quiet = quiet, stackTrace = false, }	if schemaNode.type and schemaNode.type ~= "any" then local expectedType = h.getLuaType(schemaNode.type) addIfError(utilsValidate.type(expectedType)) end if schemaNode.required then addIfError(utilsValidate.required) end if schemaNode.enum then addIfError(utilsValidate.enum(schemaNode.enum)) end return errors end

function h.errorCollector(errorTbl, value, dataName, dataPath, quiet, isKey) return function(validator) local err, msg = validator(value, dataName, dataPath, isKey, {			quiet = quiet,			stackTrace = false,		}) if err then table.insert(errorTbl, {				path = dataName .. utilsTable.path(dataPath),				msg = msg,			}) end end end

function h.getLuaType(schemaType) if schemaType == p.TYPES.array or schemaType == p.TYPES.record or schemaType == p.TYPES.map then return "table" end return schemaType end

p.Schemas = { Schema = { allOf = { {				type = "record", properties = { {						name = "definitions", desc = "Schema fragments for referencing.", type = "map", keys = { type = "string" }, values = { _ref = "#/definitions/Schema" }, }				},				additionalProperties = true, },			{				_ref = "#/definitions/Schema", },		},		definitions = { Schema = { required = true, oneOf = { { _ref = "#/definitions/Primitive Schema" }, { _ref = "#/definitions/Array Schema" }, { _ref = "#/definitions/Record Schema" }, { _ref = "#/definitions/Map Schema" }, { _ref = "#/definitions/oneOf" }, { _ref = "#/definitions/allOf" }, { _ref = "#/definitions/Reference" }, }			},			base = { type = "record", properties = { {						name = "_id", type = "string", desc = "An ID to use for referencing.", },					{						name = "required", type = "boolean", desc = "If, the value cannot be  ." },					{						name = "desc", type = "string", desc = "Description of the schema, for documentation." },				},				additionalProperties = true, },			["Primitive Schema"] = { allOf = { { _ref = "#/definitions/base" }, {						type = "record", properties = { {								name = "type", type = "string", required = true, enum = { "string", "number", "boolean", "any" }, desc = ', ,  , or  ' },							{								name = "enum", type = "array", items = { type = "any" }, desc = "An array of values that are considered acceptable.", },						},						additionalProperties = true, }				}			},			["Array Schema"] = { allOf = { { _ref = "#/definitions/base" }, {						type = "record", properties = { {								name = "type", type = "string", required = true, enum = { "array" }, desc = 'The string .' },							{								name = "items", required = true, _ref = "#/definitions/Schema", desc = "A schema that all items in the array must adhere to.", },						},						additionalProperties = true, }				}			},			name = { type = "record", properties = { {						name = "name", required = true, type = "string", desc = "The key for the record entry." },				},				additionalProperties = true, },			["Record Schema"] = { allOf = { { _ref = "#/definitions/base" }, {						type = "record", properties = { {								name = "type", type = "string", required = true, enum = { "record" }, desc = 'The string .' },							{								name = "properties", required = true, type = "array", desc = "An array of schemas for each record entry, plus an additional field  for the name of the record entry.", items = { allOf = { { _ref = "#/definitions/name" }, { _ref = "#/definitions/Schema" }, }								},							},						},						{							name = "additionalProperties", type = "boolean", desc = "If true, the record is considered valid when it has additional properties other than the ones specified in ." },						additionalProperties = true, },				},			},			["Map Schema"] = { allOf = { { _ref = "#/definitions/base" }, {						type = "record", properties = { {								name = "type", type = "string", required = true, enum = { "map" }, desc = 'The string .' },							{								name = "keys", type = "record", required = true, _ref = "#/definitions/Schema", desc = "The schema for the keys of the map." },							{								name = "values", type = "record", required = true, _ref = "#/definitions/Schema", desc = "The schema for the values of the map.", },						},						additionalProperties = true, }				}			},			["oneOf"] = { allOf = { { _ref = "#/definitions/base" }, {						type = "record", properties = { {								name = "oneOf", required = true, type = "array", desc = "An array of subschemas. The data must be valid against exactly one of them.", items = { _ref = "#/definitions/Schema" }, }						},						additionalProperties = true, }				}			},			["allOf"] = { allOf = { { _ref = "#/definitions/base" }, {						type = "record", properties = { {								name = "allOf", required = true, type = "array", desc = "An array of subschemas. The data must be valid against all of them.", items = { _ref = "#/definitions/Schema" }, },						},						additionalProperties = true, }				}			},			["Reference"] = { allOf = { { _ref = "#/definitions/base" }, {						type = "record", properties = { {								name = "_ref", required = true, type = "string", desc = "A reference to another part of the schema.", },						},						additionalProperties = true, }				}			},		},	} }

p.Documentation = { {		outputOnly = true, name = "validate", params = { {			 	name = "schemas", description = "Collection of related schemas to validate against.", },			{				name = "schemaName", description = "A key in the above table—the particular schema to validate against, which may refernce the other schemas provided.", },			{				name = "data", description = "The data to validate against the given schema.", },			{				name = "dataName", description = "A string used in warning messages to represent the data as a variable." },		},		returns = { "The name of an error category if there are validation errors, nil otherwise", "An array of paths and messages.", },		cases = { {				description = "Basic record with three fields.", args = { {						gameItem = { type = "record", properties = { {									name = "name", type = "string", required = true, },								{									name = "games", type = "array", items = { type = "string", enum = { "TLoZ", "TAoL", "ALttP", "other" }, },								},								{									name = "cost", type = "number", }							}						},					},					"gameItem", {						cost = "50 Rupees", games = { "💩" }, nonsenseField = "foo", },					"itemVariable" },				expected = { "Category:Modules with invalid data", {						{							path = 'itemVariable["name"]', msg = ' is required but is  .', },						{							path = 'itemVariable["games"][1]', msg = ' has unexpected value. The accepted values are: ', },						{							path = 'itemVariable["cost"]', msg = ' is type   but type   was expected.', },						{							path = "itemVariable", msg = "Record  has undefined properties:  ", },					},				},			},			{				description = "Map validation.", args = { {						Games = { type = "map", keys = { type = "string" }, values = { type = "string" }, },					},					"Games", {						OoA = "Oracle of Ages", OoT = 5, [3] = "A Link to the Past", },					"games" },				expected = { "Category:Modules with invalid data", {						{							path = "games", msg = " key   is type   but type   was expected.", },						{							path = 'games["OoT"]', msg = ' is type   but type   was expected.', },					}				}			},			{				description = "A schema using  and an ID-based reference.", args = { {						numberOrArray = { oneOf = { {									_id = "#num", type = "number", },								{									type = "array", items = { _ref = "#num"}, },							},						},					},					"numberOrArray", { "foo" }, "arg", },				expected = { "Category:Modules with invalid data", {						{						msg = " does not match any   sub-schemas.", path = "arg", errors = { {									{										path = "arg", msg = " is type   but type   was expected.", },								},								{									{										path = "arg[1]", msg = " is type   but type   was expected.", },								},							},						},					},				},			},			{				description = "A schema using  and   references. A simplification of Module:Documentation's schema.", args = { {						Documentation = { oneOf = { { _ref = "#/definitions/functions", }, { _ref = "#/definitions/sections", }, },							definitions = { functions = { type = "array", items = { type = "string" }, },								sections = { type = "record", properties = { {											name = "sections", type = "array", items = { _ref = "#/definitions/functions" } },									},									additionalProperties = true, },							}						},					},					"Documentation", {						"foo", sections = { {"bar", "baz"}, { "quux" }, },					},					"doc", },				expected = { "Category:Modules with invalid data", {						{							path = "doc", msg = " matches   sub-schemas   and , but must match only one.", },					}				},			},			{				description = "A schema using ", args = { {						Schema = { allOf = { { 									type = "number", required = true, },								{ 									type = "number", enum = { 0 }, }							},						},					},					"Schema", "foo", "arg", },				expected = { "Category:Modules with invalid data", {						{							msg = " does not match   sub-schemas   and  ", path = "arg", errors = { {									{										path = "arg", msg = " is type   but type   was expected.", },								},								{									{										path = "arg", msg = " is type   but type   was expected.", },									{										path = "arg", msg = " has unexpected value  . The accepted values are:  ", },								},							},						},					}				}			},		},	}, }

return p