export async function magnitTokenize(value: string, specification: Array<{ regex: RegExp, key?: string, captureGroups?: string[], groupJoinValue?: string, resultCallback?: (value: string) => Promise }>|null = null) { class MagnitTokenizerSpecification extends Array { } // key is optional because you can leave it as null in order // to skip any tokens that match for that regex. class MagnitTokenizerSpecificationItem { regex: RegExp; key?: string; captureGroups?: string[]; groupJoinValue?: string; resultCallback?: (value: string) => Promise constructor(regex: RegExp, options: { key?:string, captureGroups?: string[], groupJoinValue?:string, resultCallback?:(value:string) =>Promise } | null = null) { this.regex = regex; this.key = options?.key, this.captureGroups = options?.captureGroups; this.groupJoinValue = options?.groupJoinValue; this.resultCallback = options?.resultCallback; } } class MagnitToken { startIndex: number = 0; type?: string; value?: any; } class MagnitTokenizer { specification: MagnitTokenizerSpecification; private input?: string; private cursor: number = 0; private previousCursorPosition: number; private lookahead?: MagnitToken | null; constructor(specification: MagnitTokenizerSpecification) { this.specification = specification; this.input = ""; this.previousCursorPosition = -1; } async parse(input: string, specification: MagnitTokenizerSpecification | null = null) : Promise { if(specification != null) { this.specification = specification; } return await this.getTokens(input); } private async getTokens(input: string) : Promise { this.input = input; this.cursor = 0; this.lookahead = await this.getNextToken(); const tokens: MagnitToken[] = []; while(this.hasMoreTokens()) { this.previousCursorPosition = this.getCursorPosition(); tokens.push(await this.consume(this.lookahead?.type!)); } if(this.lookahead != null) { tokens.push(await this.consume(this.lookahead?.type!)) } return tokens; } private async consume(tokenType: string) : Promise { const token = this.lookahead; if(token == null) { throw new Error(`Unexpected end of input. Expected: "${tokenType}".`); } if(token.type != tokenType) { throw new Error(`Unexpected token type: "${token.type}". Expected: "${tokenType}".`) } this.lookahead = await this.getNextToken(); return token; } isEndOfFile(): boolean { if(this.input == null) return true; return this.cursor == this.input?.length; } hasMoreTokens(): boolean { if(this.input == null) return false; return this.cursor < this.input?.length; } getCursorPosition() : number { return this.cursor; } async getNextToken(): Promise { if(!this.hasMoreTokens() || this.specification == null || this.input == null) { return null; } let input: string = this.input.substring(this.cursor); let startIndex : number = this.cursor; for(const item of this.specification) { const tokenType: string | undefined = item.key; const tokenValue: string | null = this.match(item, input); if(tokenValue == null) { continue; } if(tokenType == null) { return await this.getNextToken(); } const token = new MagnitToken(); token.startIndex = startIndex; token.type = tokenType; token.value = (item.resultCallback == null) ? tokenValue : await item.resultCallback(tokenValue); return token; } throw new Error(`Unexpected token: "${input[0]}".`) } private match(item: MagnitTokenizerSpecificationItem, input : string) : string | null { let match = input.match(item.regex); if(match == null || match.length == 0) { return null; } let value: string = match[0]; this.cursor += value.length; if(item.captureGroups != null && match.groups != null) { let combinedCaptureValue: string = ""; for(let i = 0; i < item.captureGroups.length; i++) { const captureGroupName: string = item.captureGroups[i]; const capturedValue = match.groups[captureGroupName]; if(capturedValue != null) { combinedCaptureValue += capturedValue; if(item.groupJoinValue !+ null && item.captureGroups.length > (i + 1)) { combinedCaptureValue += item.groupJoinValue; } } } value = combinedCaptureValue; } return value; } } // regex is checked against string in the order that it is defined in the specification // whitespace first, then comments, then good matches, then a utility match to catch any // unknown characters. const defaultSpecification = [ // Whitespace new MagnitTokenizerSpecificationItem(new RegExp(/^\s+/)), // undefined/null as value; matches will be skipped. // Comments new MagnitTokenizerSpecificationItem(new RegExp(/^\/\/.*/)), // undefined/null as value; matches will be skipped. new MagnitTokenizerSpecificationItem(new RegExp(/^\/\*[\s\S]*?\*\//)), // undefined/null as value; matches will be skipped. // Variables new MagnitTokenizerSpecificationItem( new RegExp(/^\$[^(\/\/)(\/\*)]*/), { key:"VARIABLE", // a unique name to use as a label for all matching items. resultCallback: async (result: string) => { // mutate the result to fit your needs let value = result; if(value.startsWith('$')) { value = value.substring(1); } return value; } } ), // Utility new MagnitTokenizerSpecificationItem(new RegExp(/^[\s\S]*/), { key:"UNKNOWN" }), ]; if(specification == null) { specification = defaultSpecification; } const tokenizer = new MagnitTokenizer(specification); return await tokenizer.parse(value); }

testTokenizer