I grabbed this code from gist.github.com/pmachapman/661f0fff9814231fde48 and "improved" it.
tinybasic.c is an extended BASIC including break, continue, and while statements. (952 SLOC)
tinybasictok.c adds minimal hacks to pseudo-tokenize. (1093 SLOC)
The "tokenized" version runs about 6x faster. For instance, on an x86 processor, one iteration of the BASIC program "for i=1to 1000000000next" is executed in 31.5 clock cycles.
Here is a debugger trace of one iteration:
1203 token_type = get_token();
get_token () at tinybasictok.c:174
174 lastTokPos = prog;
176 if (*prog & 0x80) { /* tokenized action */
177 int bytecode = *prog & 0x7f;
179 if (bytecode < FINISHED) {
181 tok = bytecode;
182 if (tok != EOL) {
183 token_type = COMMAND;
184 prog += prog[1];
281 return token_type;
282 }
main (argc=2, argv=0x7fffffffe168) at tinybasictok.c:1203
1203 token_type = get_token();
1205 if(token_type==VARIABLE) {
1209 switch(tok) {
1211 next();
next () at tinybasictok.c:752
752 struct loop_stack *ii = &lstack[ltos];
754 if (ltos == -1 || ii->var == 0xff) {
758 if (ii->endLoc == 0) { /* allow for fast break-stmt */
773 ++variables[ii->var]; /* increment control variable */
774 if(variables[ii->var] <= ii->target) {
775 prog = ii->beginLoc; /* loop */
788 }
main (argc=2, argv=0x7fffffffe168) at tinybasictok.c:1212
1212 break;
1254 } while (tok != FINISHED);