function tokenize(program) { var delimiter = null; var tokens = []; var current_token = []; // Doing this for var STATE_DELIM = "detect the delimiter"; var STATE_COLLECT_TOKENS = "collecting the tokens"; var STATE_RAW_STRING = "collecting a raw string"; var state = "detect the delimiter"; function is_raw_start(idx) { return program[idx] === "[" && program[idx+1] === "=" && program[idx+2] == "["; } function is_raw_end(idx) { return program[idx] === "]" && program[idx+1] === "=" && program[idx+2] == "]"; } for (var i = 0; i < program.length; i++) { var current_character = program[i]; if (state == STATE_DELIM) { if (!is_whitespace(current_character)) { delimiter = current_character; tokens.push(delimiter); state = STATE_COLLECT_TOKENS; } } else if (state == STATE_COLLECT_TOKENS) { if (is_raw_start(i)) { i += 2; // The i++ at the top of the loop will take care of the rest state = STATE_RAW_STRING; if (current_token.length > 0) { tokens.push(current_token.join("")) current_token = []; } continue; } if (is_whitespace(current_character) || is_default_delimiter(current_character) || current_character == delimiter) { if (current_token.length != 0) { tokens.push(current_token.join("")); current_token = []; } if (!is_whitespace(current_character)) { tokens.push(current_character); } } else { current_token.push(current_character); } } else if (state == STATE_RAW_STRING) { if (is_raw_end(i)) { state = STATE_COLLECT_TOKENS; i+=2; tokens.push(current_token.join("")); current_token = []; continue; } current_token.push(current_character); } } if (current_token.length > 0) { tokens.push(current_token.join("")); } return {delimiter: delimiter, tokens: tokens}; }