Sol Part 64: Now with Less Non-Deterministic "Electric" Strangs!

Pursuant some fuzzing, *most* places where a SOL_STRING (C string)
escaped to the user are now replaced with SOL_BUFFER. This allows,
amongst other things, source strings to contain embedded NUL characters.

Also, a fix for comma-after-empty-list in the parser is implemented.

.gitignore

@@ -15,3 +15,4 @@ doxyxml
 valgrind.log
 *_INFO
 prof
+/afl

Makefile

@@ -67,7 +67,7 @@ $(LINKED_VERS): sol$(SOL_VER)
 	rm $@; ln -s $? $@
 	
 sol$(SOL_VER): $(OBJ)
-	$(CC) $(CFLAGS) $? $(LDFLAGS) -o $@
+	$(CC) $(CFLAGS) $^ $(LDFLAGS) -o $@
 
 test: all $(sort $(patsubst tests/%.sol,test_%,$(wildcard tests/*.sol))) $(sort $(patsubst tests/%.sol,testcomp_%,$(wildcard tests/*.sol)))
 

RECIPES

@@ -6,3 +6,6 @@ make CC=clang CFLAGS="-fprofile-instr-generate -fcoverage-mapping"
 
 use Boehm GC where available:
 make LDFLAGS=-lgc
+
+AFL fuzzing (use afl-fuzz on the resulting instrumented binary):
+make CC=afl-gcc

ast.h

@@ -24,7 +24,7 @@ typedef struct tag_stmt_node stmt_node;
  *
  * Defines the types of literals that may appear in a source program.
  */
-typedef enum {LIT_INT=1024, LIT_FLOAT, LIT_STRING, LIT_NONE} lit_t;
+typedef enum {LIT_INT=1024, LIT_FLOAT, LIT_STRING, LIT_BUFFER, LIT_NONE} lit_t;
 /** Literal node
  *
  * Represents a literal in a source program.
@@ -35,9 +35,15 @@ typedef struct {
 		long ival; ///< Integer value for `LIT_INT`.
 		double fval; ///< Floating-point value for `LIT_FLOAT`.
 		char *str; ///< String value for `LIT_STRING`.
+		unsigned long *buf; ///< Buffer value for `LIT_BUFFER`; points to. (char *)(buf + 1) points to the first byte in the buffer. There are *buf bytes starting there. See also LENGTH_OF and BYTES_OF.
 	};
 } lit_node;
 
+/** Returns the length (as an unsigned long) of the buffer in bytes, not including the length itself. */
+#define LENGTH_OF(buf) (*((unsigned long *) (buf)))
+/** Returns a (char *) pointing to the first byte in the buffer. */
+#define BYTES_OF(buf) ((char *) (((unsigned long *) (buf)) + 1))
+
 /** Binary operation type
  *
  * Defines the types of binary operators that may occur in a source program.
@@ -237,10 +243,12 @@ typedef enum {
 	BC_LIT_INT,
 	BC_LIT_FLOAT,
 	BC_LIT_STRING,
+	BC_LIT_BUFFER,
 	BC_LIT_NONE,
 	BC_INT,
 	BC_FLOAT,
 	BC_STRING,
+	BC_BUFFER,
 	BC_LIST_ST,
 	BC_LIST_EX,
 	BC_LIST_AS,
@@ -333,6 +341,7 @@ void sol_ser_lit(FILE *, lit_node *);
 void sol_ser_int(FILE *, long);
 void sol_ser_float(FILE *, double);
 void sol_ser_str(FILE *, const char *);
+void sol_ser_buf(FILE *, unsigned long *);
 
 void *sol_deser(FILE *);
 void *sol_deser_checked(FILE *, bytecode);

astprint.c

@@ -82,6 +82,10 @@ void prex(sol_state_t *state, expr_node *node, int lev) {
 					prlev(state, lev, "String: %s", node->lit->str);
 					break;
 
+				case LIT_BUFFER:
+					prlev(state, lev, "Buffer of %lu bytes:", LENGTH_OF(node->lit->buf));
+					prlev(state, lev + 1, "%s", BYTES_OF(node->lit->buf));
+
 				case LIT_NONE:
 					prlev(state, lev, "None");
 					break;

builtins.c

@@ -1,3 +1,4 @@
+#define _GNU_SOURCE
 #include <string.h>
 #include <stdlib.h>
 #include <stdio.h>
@@ -59,6 +60,14 @@ sol_object_t *sol_f_default_repr(sol_state_t *state, sol_object_t *args) {
 	return res;
 }
 
+sol_object_t *sol_f_default_tobuffer(sol_state_t *state, sol_object_t *args) {
+	sol_object_t *obj = sol_list_get_index(state, args, 0), *str = CALL_METHOD(state, obj, tostring, args);
+	sol_object_t *res = sol_new_buffer(state, strdup(str->str), strlen(str->str), OWN_FREE, NULL, NULL);
+	sol_obj_free(obj);
+	sol_obj_free(str);
+	return res;
+}
+
 sol_object_t *sol_f_tbang(sol_state_t *state, sol_object_t *args) {
 	sol_object_t *a = sol_list_get_index(state, args, 0), *b = sol_list_get_index(state, args, 1);
 	int refa = a->refcnt, refb = b->refcnt;
@@ -100,6 +109,13 @@ sol_object_t *sol_f_tostring(sol_state_t *state, sol_object_t *args) {
 	return res;
 }
 
+sol_object_t *sol_f_tobuffer(sol_state_t *state, sol_object_t *args) {
+	sol_object_t *obj = sol_list_get_index(state, args, 0);
+	sol_object_t *res = CALL_METHOD(state, obj, tobuffer, args);
+	sol_obj_free(obj);
+	return res;
+}
+
 sol_object_t *sol_f_try(sol_state_t *state, sol_object_t *args) {
 	sol_object_t *func = sol_list_get_index(state, args, 0), *fargs = sol_list_sublist(state, args, 1);
 	sol_object_t *ls = sol_new_list(state), *one = sol_new_int(state, 1);
@@ -150,7 +166,7 @@ sol_object_t *sol_f_error(sol_state_t *state, sol_object_t *args) {
 
 sol_object_t *sol_f_type(sol_state_t *state, sol_object_t *args) {
 	sol_object_t *obj = sol_list_get_index(state, args, 0);
-	sol_object_t *res = sol_new_string(state, obj->ops->tname);
+	sol_object_t *res = sol_new_buffer(state, strdup(obj->ops->tname), strlen(obj->ops->tname), OWN_FREE, NULL, NULL);
 	sol_obj_free(obj);
 	return res;
 }
@@ -252,11 +268,14 @@ void ob_print(sol_object_t *obj) {
 			break;
 
 		case SOL_BUFFER:
+			/*
 			if(obj->sz == -1) {
 				printf("<Buffer @%p>", obj->buffer);
 			} else {
 				printf("<Buffer @%p size %ld>", obj->buffer, obj->sz);
 			}
+			*/
+			fwrite(obj->buffer, sizeof(char), obj->sz, stdout);
 			break;
 
 		case SOL_CDATA:
@@ -447,7 +466,7 @@ sol_object_t *sol_f_ord(sol_state_t *state, sol_object_t *args) {
 sol_object_t *sol_f_chr(sol_state_t *state, sol_object_t *args) {
 	sol_object_t *arg = sol_list_get_index(state, args, 0), *iarg = sol_cast_int(state, arg);
 	char cbuf[2] = {iarg->ival, 0};
-	sol_object_t *res = sol_new_string(state, cbuf);
+	sol_object_t *res = sol_new_buffer(state, strdup(cbuf), 1, OWN_FREE, NULL, NULL);
 	sol_obj_free(arg);
 	sol_obj_free(iarg);
 	return res;
@@ -488,6 +507,38 @@ sol_object_t *sol_f_debug_scopes(sol_state_t *state, sol_object_t *args) {
 	return sol_incref(state->scopes);
 }
 
+sol_object_t *sol_f_debug_getops(sol_state_t *state, sol_object_t *args) {
+	sol_object_t *obj = sol_list_get_index(state, args, 0), *res = sol_new_map(state);
+	sol_map_borrow_name(state, res, "add", sol_new_cfunc(state, obj->ops->add, "add"));
+	sol_map_borrow_name(state, res, "sub", sol_new_cfunc(state, obj->ops->sub, "sub"));
+	sol_map_borrow_name(state, res, "mul", sol_new_cfunc(state, obj->ops->mul, "mul"));
+	sol_map_borrow_name(state, res, "div", sol_new_cfunc(state, obj->ops->div, "div"));
+	sol_map_borrow_name(state, res, "mod", sol_new_cfunc(state, obj->ops->mod, "mod"));
+	sol_map_borrow_name(state, res, "pow", sol_new_cfunc(state, obj->ops->pow, "pow"));
+	sol_map_borrow_name(state, res, "tbang", sol_new_cfunc(state, obj->ops->tbang, "tbang"));
+	sol_map_borrow_name(state, res, "band", sol_new_cfunc(state, obj->ops->band, "band"));
+	sol_map_borrow_name(state, res, "bor", sol_new_cfunc(state, obj->ops->bor, "bor"));
+	sol_map_borrow_name(state, res, "bxor", sol_new_cfunc(state, obj->ops->bxor, "bxor"));
+	sol_map_borrow_name(state, res, "blsh", sol_new_cfunc(state, obj->ops->blsh, "blsh"));
+	sol_map_borrow_name(state, res, "brsh", sol_new_cfunc(state, obj->ops->brsh, "brsh"));
+	sol_map_borrow_name(state, res, "bnot", sol_new_cfunc(state, obj->ops->bnot, "bnot"));
+	sol_map_borrow_name(state, res, "cmp", sol_new_cfunc(state, obj->ops->cmp, "cmp"));
+	sol_map_borrow_name(state, res, "call", sol_new_cfunc(state, obj->ops->call, "call"));
+	sol_map_borrow_name(state, res, "index", sol_new_cfunc(state, obj->ops->index, "index"));
+	sol_map_borrow_name(state, res, "setindex", sol_new_cfunc(state, obj->ops->setindex, "setindex"));
+	sol_map_borrow_name(state, res, "len", sol_new_cfunc(state, obj->ops->len, "len"));
+	sol_map_borrow_name(state, res, "iter", sol_new_cfunc(state, obj->ops->iter, "iter"));
+	sol_map_borrow_name(state, res, "toint", sol_new_cfunc(state, obj->ops->toint, "toint"));
+	sol_map_borrow_name(state, res, "tofloat", sol_new_cfunc(state, obj->ops->tofloat, "tofloat"));
+	sol_map_borrow_name(state, res, "tostring", sol_new_cfunc(state, obj->ops->tostring, "tostring"));
+	sol_map_borrow_name(state, res, "tobuffer", sol_new_cfunc(state, obj->ops->tobuffer, "tobuffer"));
+	sol_map_borrow_name(state, res, "repr", sol_new_cfunc(state, obj->ops->repr, "repr"));
+	sol_map_borrow_name(state, res, "init", sol_new_cfunc(state, obj->ops->init, "init"));
+	sol_map_borrow_name(state, res, "free", sol_new_cfunc(state, obj->ops->free, "free"));
+	sol_obj_free(obj);
+	return res;
+}
+
 #ifndef NO_READLINE
 sol_object_t *sol_f_readline_readline(sol_state_t *state, sol_object_t *args) {
 	sol_object_t *obj, *objstr, *res;
@@ -502,10 +553,10 @@ sol_object_t *sol_f_readline_readline(sol_state_t *state, sol_object_t *args) {
 		line = readline("");
 	}
 	if(line) {
-		res = sol_new_string(state, line);
+		res = sol_new_buffer(state, strdup(line), strlen(line), OWN_FREE, NULL, NULL);
 		free(line);
 	} else {
-		res = sol_new_string(state, "");
+		res = sol_new_buffer(state, strdup(""), 0, OWN_FREE, NULL, NULL);
 	}
 	return res;
 }
@@ -550,6 +601,31 @@ sol_object_t *sol_f_iter_str(sol_state_t *state, sol_object_t *args) {
 	return res;
 }
 
+sol_object_t *sol_f_iter_buffer(sol_state_t *state, sol_object_t *args) {
+	sol_object_t *obj = sol_list_get_index(state, args, 0), *local = sol_list_get_index(state, args, 1);
+	sol_object_t *idx = sol_map_get_name(state, local, "idx"), *sz = sol_map_get_name(state,local, "sz"), *res;
+	if(sol_is_none(state, idx) || sol_is_none(state, sz)) {
+		sol_obj_free(idx);
+		sol_obj_free(sz);
+		idx = sol_new_buffer(state, (void *) 0, sizeof(void *), OWN_NONE, NULL, NULL);
+		sz = sol_new_int(state, obj->sz);
+		sol_map_set_name(state, local, "idx", idx);
+		sol_map_set_name(state, local, "sz", sz);
+	}
+	if(((size_t) idx->buffer) >= sz->ival) {
+		sol_obj_free(idx);
+		sol_obj_free(sz);
+		sol_obj_free(local);
+		return sol_incref(state->None);
+	}
+	res = sol_new_buffer(state, ((char *) obj->buffer) + ((size_t) idx->buffer), 1, OWN_NONE, NULL, NULL);
+	idx->buffer = (void *) ((size_t) idx->buffer + 1);
+	sol_obj_free(idx);
+	sol_obj_free(sz);
+	sol_obj_free(local);
+	return res;
+}
+
 sol_object_t *sol_f_iter_list(sol_state_t *state, sol_object_t *args) {
 	sol_object_t *obj = sol_list_get_index(state, args, 0), *local = sol_list_get_index(state, args, 1);
 	sol_object_t *index = sol_map_get_name(state, local, "idx"), *res;
@@ -611,6 +687,12 @@ sol_object_t *sol_f_singlet_tostring(sol_state_t *state, sol_object_t *args) {
 	return res;
 }
 
+sol_object_t *sol_f_singlet_tobuffer(sol_state_t *state, sol_object_t *args) {
+	sol_object_t *obj = sol_list_get_index(state, args, 0), *res = sol_new_buffer(state, strdup(obj->str), strlen(obj->str), OWN_FREE, NULL, NULL);
+	sol_obj_free(obj);
+	return res;
+}
+
 sol_object_t *sol_f_int_add(sol_state_t *state, sol_object_t *args) {
 	sol_object_t *a = sol_list_get_index(state, args, 0), *b = sol_list_get_index(state, args, 1), *bint = sol_cast_int(state, b);
 	sol_object_t *res = sol_new_int(state, a->ival + bint->ival);
@@ -911,7 +993,12 @@ sol_object_t *sol_f_str_mul(sol_state_t *state, sol_object_t *args) {
 
 sol_object_t *sol_f_str_cmp(sol_state_t *state, sol_object_t *args) {
     sol_object_t *a = sol_list_get_index(state, args, 0), *b = sol_list_get_index(state, args, 1);
-    sol_object_t *res;
+    sol_object_t *sb, *res;
+	if(sol_is_buffer(b)) {
+		sb = sol_cast_string(state, b);
+		sol_obj_free(b);
+		b = sb;
+	}
 	if(sol_is_string(b)) {
 		res = sol_new_int(state, strcmp(a->str, b->str));
 	} else {
@@ -970,6 +1057,13 @@ sol_object_t *sol_f_str_tostring(sol_state_t *state, sol_object_t *args) {
 	return sol_list_get_index(state, args, 0);
 }
 
+sol_object_t *sol_f_str_tobuffer(sol_state_t *state, sol_object_t *args) {
+	sol_object_t *str = sol_list_get_index(state, args, 0);
+	sol_object_t *res = sol_new_buffer(state, strdup(str->str), strlen(str->str), OWN_FREE, NULL, NULL);
+	sol_obj_free(str);
+	return res;
+}
+
 sol_object_t *sol_f_str_repr(sol_state_t *state, sol_object_t *args) {
 	sol_object_t *obj = sol_list_get_index(state, args, 0), *cur = sol_new_string(state, "\""), *next = sol_string_concat(state, cur, obj);
 	sol_obj_free(cur);
@@ -1145,7 +1239,7 @@ sol_object_t *sol_f_list_cmp(sol_state_t *state, sol_object_t *args) {
 sol_object_t *sol_f_list_index(sol_state_t *state, sol_object_t *args) {
 	sol_object_t *ls = sol_list_get_index(state, args, 0), *b = sol_list_get_index(state, args, 1), *ival;
 	sol_object_t *res, *funcs;
-	if(sol_is_string(b)) {
+	if(sol_is_name(b)) {
 		funcs = sol_get_methods_name(state, "list");
 		res = sol_map_get(state, funcs, b);
 		sol_obj_free(funcs);
@@ -1515,35 +1609,35 @@ sol_object_t *sol_f_func_index(sol_state_t *state, sol_object_t *args) {
 	sol_object_t *func = sol_list_get_index(state, args, 0), *key = sol_list_get_index(state, args, 1), *res;
 	identlist_node *curi;
 	int i = 0;
-	if(!sol_is_string(key)) {
+	if(!sol_is_name(key)) {
 		res = sol_map_get(state, func->udata, key);
 	} else {
-		if(sol_string_eq(state, key, "name")) {
+		if(sol_name_eq(state, key, "name")) {
 			if(func->fname) {
 				res = sol_new_string(state, func->fname);
 			} else {
 				res = sol_incref(state->None);
 			}
-		} else if(sol_string_eq(state, key, "closure")) {
+		} else if(sol_name_eq(state, key, "closure")) {
 			res = sol_incref(func->closure);
-		} else if(sol_string_eq(state, key, "udata")) {
+		} else if(sol_name_eq(state, key, "udata")) {
 			res = sol_incref(func->udata);
-		} else if(sol_string_eq(state, key, "stmt")) {
+		} else if(sol_name_eq(state, key, "stmt")) {
 			res = sol_new_stmtnode(state, st_copy((stmt_node *) func->func));
-		} else if(sol_string_eq(state, key, "args")) {
+		} else if(sol_name_eq(state, key, "args")) {
 			res = sol_new_list(state);
 			curi = func->args;
 			while(curi) {
 				sol_list_insert(state, res, i++, sol_new_string(state, curi->ident));
 				curi = curi->next;
 			}
-		} else if(sol_string_eq(state, key, "rest")) {
+		} else if(sol_name_eq(state, key, "rest")) {
 			if(func->rest) {
 				res = sol_new_string(state, func->rest);
 			} else {
 				res = sol_incref(state->None);
 			}
-		} else if(sol_string_eq(state, key, "annos")) {
+		} else if(sol_name_eq(state, key, "annos")) {
 			res = sol_incref(func->annos);
 		} else {
 			res = sol_map_get(state, func->udata, key);
@@ -1558,21 +1652,25 @@ sol_object_t *sol_f_func_setindex(sol_state_t *state, sol_object_t *args) {
 	sol_object_t *func = sol_list_get_index(state, args, 0), *key = sol_list_get_index(state, args, 1), *val = sol_list_get_index(state, args, 2), *temp, *str;
 	size_t i, len;
 	identlist_node *cur, *prev;
-	if(sol_string_eq(state, key, "name") && sol_is_string(val)) {
+	if(sol_name_eq(state, key, "name") && sol_is_name(val)) {
 		free(func->fname);
-		func->fname = strdup(val->str);
-	} else if(sol_string_eq(state, key, "closure") && sol_is_map(val)) {
+		if(sol_is_string(val)) {
+			func->fname = strdup(val->str);
+		} else {
+			func->fname = sol_buffer_strdup(val);
+		}
+	} else if(sol_name_eq(state, key, "closure") && sol_is_map(val)) {
 		temp = func->closure;
 		func->closure = sol_incref(val);
 		sol_obj_free(temp);
-	} else if(sol_string_eq(state, key, "udata") && sol_is_map(val)) {
+	} else if(sol_name_eq(state, key, "udata") && sol_is_map(val)) {
 		temp = func->udata;
 		func->udata = sol_incref(val);
 		sol_obj_free(temp);
-	} else if(sol_string_eq(state, key, "stmt") && sol_is_aststmt(val)) {
+	} else if(sol_name_eq(state, key, "stmt") && sol_is_aststmt(val)) {
 		st_free(func->func);
 		func->func = st_copy(val->node);
-	} else if(sol_string_eq(state, key, "args") && sol_is_list(val)) {
+	} else if(sol_name_eq(state, key, "args") && sol_is_list(val)) {
 		idl_free(func->args);
 		func->args = NEW(identlist_node);
 		cur = func->args;
@@ -1591,10 +1689,14 @@ sol_object_t *sol_f_func_setindex(sol_state_t *state, sol_object_t *args) {
 		prev->next = NULL;
 		if(cur == func->args) func->args = NULL;
 		free(cur);
-	} else if(sol_string_eq(state, key, "rest") && sol_is_string(val)) {
+	} else if(sol_name_eq(state, key, "rest") && sol_is_name(val)) {
 		free(func->rest);
-		func->rest = strdup(val->str);
-	} else if(sol_string_eq(state, key, "annos") && sol_is_map(val)) {
+		if(sol_is_string(val)) {
+			func->rest = strdup(val->str);
+		} else {
+			func->rest = sol_buffer_strdup(val);
+		}
+	} else if(sol_name_eq(state, key, "annos") && sol_is_map(val)) {
 		sol_obj_free(func->annos);
 		func->annos = sol_incref(val);
 	} else {
@@ -2307,26 +2409,158 @@ sol_object_t *sol_f_astnode_tostring(sol_state_t *state, sol_object_t *args) {
 }
 
 sol_object_t *sol_f_buffer_index(sol_state_t *state, sol_object_t *args) {
+	sol_object_t *a = sol_list_get_index(state, args, 0);
 	sol_object_t *key = sol_list_get_index(state, args, 1), *funcs = sol_get_methods_name(state, "buffer");
-	sol_object_t *res = sol_map_get(state, funcs, key);
+	sol_object_t *res;
+	if(sol_is_name(key)) {
+		res = sol_map_get(state, funcs, key);
+	} else if(sol_is_int(key)) {
+		res = sol_new_buffer(state, a->buffer, (a->sz < 0) ? a->sz : (a->sz - key->ival), OWN_NONE, NULL, NULL);
+	} else {
+		res = sol_f_not_impl(state, args);
+	}
+	sol_obj_free(a);
 	sol_obj_free(key);
 	sol_obj_free(funcs);
 	return res;
 }
 
+sol_object_t *sol_f_buffer_add(sol_state_t *state, sol_object_t *args) {
+	sol_object_t *a = sol_list_get_index(state, args, 0), *b = sol_list_get_index(state, args, 1), *bb = sol_cast_buffer(state, b);
+	sol_object_t *res = sol_buffer_concat(state, a, bb);
+	sol_obj_free(a);
+	sol_obj_free(b);
+	if(sol_has_error(state)) {
+		sol_obj_free(res);
+		return sol_incref(state->None);
+	}
+	return res;
+}
+
+sol_object_t *sol_f_buffer_mul(sol_state_t *state, sol_object_t *args) {
+	sol_object_t *a = sol_list_get_index(state, args, 0), *b = sol_list_get_index(state, args, 1), *bint = sol_cast_int(state, b);
+	long ival, i;
+	size_t sz;
+	char *buf;
+	sol_obj_free(b);
+	if(sol_has_error(state)) {
+		sol_obj_free(a);
+		sol_obj_free(bint);
+		return sol_incref(state->None);
+	}
+	ival = bint->ival;
+	sol_obj_free(bint);
+	if(a->sz < 0) {
+		sol_obj_free(a);
+		return sol_set_error_string(state, "Multiply unsized buffer");
+	}
+	if(ival < 0) {
+		ival = 0;
+	}
+	sz = a->sz * ival;
+	buf = malloc(sz * sizeof(char));
+	for(i = 0; i < ival; i++) {
+		memcpy(buf + (i * a->sz), a->buffer, a->sz);
+	}
+	sol_obj_free(a);
+	return sol_new_buffer(state, buf, sz, OWN_FREE, NULL, NULL);
+}
+
+sol_object_t *sol_f_buffer_cmp(sol_state_t *state, sol_object_t *args) {
+	sol_object_t *a = sol_list_get_index(state, args, 0), *b = sol_list_get_index(state, args, 1);
+	sol_object_t *bb, *res;
+	if(sol_is_string(b)) {
+		bb = sol_cast_buffer(state, b);
+		sol_obj_free(b);
+		b = bb;
+	}
+	if(sol_is_buffer(b)) {
+		ssize_t len = a->sz;
+		if(a->sz >= 0 && b->sz >= 0 && a->sz != b->sz) {
+			res = sol_new_int(state, 1);
+		} else {
+			if(len < 0) {
+				len = b->sz;
+			}
+			if(a->sz >= 0 && len > a->sz) len = a->sz;
+			if(b->sz >= 0 && len > b->sz) len = b->sz;
+			if(len < 0) {
+				res = sol_new_int(state, 1);
+			} else {
+				res = sol_new_int(state, memcmp(a->buffer, b->buffer, len));
+			}
+		}
+	} else {
+		res = sol_new_int(state, 1);
+	}
+	sol_obj_free(a);
+	sol_obj_free(b);
+	return res;
+}
+
+sol_object_t *sol_f_buffer_len(sol_state_t *state, sol_object_t *args) {
+	sol_object_t *a = sol_list_get_index(state, args, 0);
+	sol_object_t *res = sol_new_int(state, a->sz);
+	sol_obj_free(a);
+	return res;
+}
+
+sol_object_t *sol_f_buffer_iter(sol_state_t *state, sol_object_t *args) {
+	return sol_new_cfunc(state, sol_f_iter_buffer, "iter.buffer");
+}
+
 sol_object_t *sol_f_buffer_tostring(sol_state_t *state, sol_object_t *args) {
 	sol_object_t *buf = sol_list_get_index(state, args, 0), *res;
+	char *b;
+	/*
 	char s[64];
 	if(buf->sz == -1) {
 		snprintf(s, 64, "<Buffer @%p>", buf->buffer);
 	} else {
-		snprintf(s, 64, "<Buffer @%p size %ld", buf->buffer, buf->sz);
+		snprintf(s, 64, "<Buffer @%p size %ld>", buf->buffer, buf->sz);
+	}
+	*/
+	if(buf->sz < 0) {
+		res = sol_new_string(state, "<UNSIZED_BUFFER>");
+	} else {
+		b = malloc(buf->sz + 1);
+		strncpy(b, buf->buffer, buf->sz);
+		b[buf->sz] = '\0';
+		res = sol_new_string(state, b);
+		free(b);
 	}
-	res = sol_new_string(state, s);
 	sol_obj_free(buf);
 	return res;
 }
 
+sol_object_t *sol_f_buffer_repr(sol_state_t *state, sol_object_t *args) {
+	sol_object_t *obj = sol_list_get_index(state, args, 0), *cur = sol_new_buffer(state, "'", 1, OWN_NONE, NULL, NULL), *next = sol_buffer_concat(state, cur, obj);
+	sol_object_t *termin = sol_new_buffer(state, "'", 2, OWN_NONE, NULL, NULL);
+	sol_obj_free(cur);
+	cur = next;
+	next = sol_buffer_concat(state, cur, termin);
+	sol_obj_free(cur);
+	return next;
+}
+
+sol_object_t *sol_f_buffer_toint(sol_state_t *state, sol_object_t *args) {
+	sol_object_t *a = sol_list_get_index(state, args, 0);
+	sol_object_t *res = sol_new_int(state, atoi(a->buffer));
+	sol_obj_free(a);
+	return res;
+}
+
+sol_object_t *sol_f_buffer_tofloat(sol_state_t *state, sol_object_t *args) {
+	sol_object_t *a = sol_list_get_index(state, args, 0);
+	sol_object_t *res = sol_new_float(state, atof(a->buffer));
+	sol_obj_free(a);
+	return res;
+}
+
+sol_object_t *sol_f_buffer_tobuffer(sol_state_t *state, sol_object_t *args) {
+	return sol_list_get_index(state, args, 0);
+}
+
 sol_object_t *sol_f_buffer_new(sol_state_t *state, sol_object_t *args) {
 	sol_object_t *sz = sol_list_get_index(state, args, 0), *isz = sol_cast_int(state, sz);
 	size_t bufsz = isz->ival;
@@ -2585,6 +2819,97 @@ sol_object_t *sol_f_buffer_size(sol_state_t *state, sol_object_t *args) {
 	return res;
 }
 
+sol_object_t *sol_f_buffer_sub(sol_state_t *state, sol_object_t *args) {
+	sol_object_t *buf = sol_list_get_index(state, args, 0);
+	sol_object_t *low = sol_list_get_index(state, args, 1), *high = sol_list_get_index(state, args, 2);
+	sol_object_t *ilow, *ihigh;
+	long l, h;
+	char *b;
+	if(sol_is_none(state, low)) {
+		ilow = sol_new_int(state, 0);
+	} else {
+		ilow = sol_cast_int(state, low);
+	}
+	if(sol_is_none(state, high)) {
+		ihigh = sol_new_int(state, buf->sz);
+	} else {
+		ihigh = sol_cast_int(state, high);
+	}
+	l = ilow->ival;
+	h = ihigh->ival;
+	sol_obj_free(low);
+	sol_obj_free(high);
+	sol_obj_free(ilow);
+	sol_obj_free(ihigh);
+	if(l < 0) {
+		l += buf->sz;
+		if(l < 0) {
+			l = 0;
+		}
+	}
+	if(l > buf->sz) {
+		l = buf->sz;
+	}
+	if(h < 0) {
+		h += buf->sz;
+		if(h < 0) {
+			h = 0;
+		}
+	}
+	if(h > buf->sz) {
+		h = buf->sz;
+	}
+	if(l >= h) {
+		sol_obj_free(buf);
+		return sol_new_buffer(state, NULL, 0, OWN_NONE, NULL, NULL);
+	}
+	b = malloc(sizeof(char) * (h - l));
+	memcpy(b, buf->buffer, h - l);
+	sol_obj_free(buf);
+	return sol_new_buffer(state, b, h - l, OWN_FREE, NULL, NULL);
+}
+
+sol_object_t *sol_f_buffer_split(sol_state_t *state, sol_object_t *args) {
+	sol_object_t *buf = sol_list_get_index(state, args, 0);
+	char *b;
+	sol_object_t *str, *res, *ls;
+	if(buf->sz < 0) {
+		sol_obj_free(buf);
+		return sol_set_error_string(state, "split unsized buffer");
+	}
+	b = malloc(sizeof(char) * (buf->sz + 1));
+	memcpy(b, buf->buffer, buf->sz);
+	b[buf->sz] = '\0';
+	str = sol_new_string(state, b);
+	free(b);
+	ls = sol_new_list(state);
+	sol_list_insert(state, ls, 0, str);
+	sol_obj_free(str);
+	str = sol_list_get_index(state, args, 1);
+	sol_list_insert(state, ls, 1, str);
+	sol_obj_free(str);
+	res = sol_f_str_split(state, ls);
+	sol_obj_free(ls);
+	return res;
+}
+
+sol_object_t *sol_f_buffer_find(sol_state_t *state, sol_object_t *args) {
+	sol_object_t *buf = sol_list_get_index(state, args, 0), *subbuf = sol_list_get_index(state, args, 1), *bsubbuf = sol_cast_buffer(state, subbuf);
+	sol_object_t *res;
+	char *ptr;
+	sol_obj_free(subbuf);
+	if(buf->sz < 0 || bsubbuf->sz < 0) {
+		sol_obj_free(buf);
+		sol_obj_free(bsubbuf);
+		return sol_set_error_string(state, "find with unsized buffer");
+	}
+	ptr = memmem(buf->buffer, buf->sz, bsubbuf->buffer, bsubbuf->sz);
+	res = sol_new_int(state, ptr ? (ptr - ((char *) buf->buffer)) : -1);
+	sol_obj_free(buf);
+	sol_obj_free(bsubbuf);
+	return res;
+}
+
 sol_object_t *sol_f_buffer_fromstring(sol_state_t *state, sol_object_t *args) {
 	sol_object_t *val = sol_list_get_index(state, args, 0), *sval = sol_cast_string(state, val);
 	size_t sz = strlen(sval->str) + 1;
@@ -2654,7 +2979,7 @@ sol_object_t *sol_f_stream_blsh(sol_state_t *state, sol_object_t *args) {
 }
 
 sol_object_t *sol_f_stream_brsh(sol_state_t *state, sol_object_t *args) {
-	return sol_f_stream_read(state, args);
+	return sol_f_stream_read_buffer(state, args);
 }
 
 sol_object_t *sol_f_stream_index(sol_state_t *state, sol_object_t *args) {
@@ -2670,10 +2995,16 @@ sol_object_t *sol_f_stream_tostring(sol_state_t *state, sol_object_t *args) {
 }
 
 sol_object_t *sol_f_stream_write(sol_state_t *state, sol_object_t *args) {
-	sol_object_t *stream = sol_list_get_index(state, args, 0), *obj = sol_list_get_index(state, args, 1), *str = sol_cast_string(state, obj);
-	size_t sz = sol_stream_printf(state, stream, "%s", str->str);
+	sol_object_t *stream = sol_list_get_index(state, args, 0), *obj = sol_list_get_index(state, args, 1), *str;
+	size_t sz;
+	if(sol_is_buffer(obj)) {
+		sz = sol_stream_fwrite(state, stream, obj->buffer, sizeof(char), obj->sz);
+	} else {
+		str = sol_cast_string(state, obj);
+		sz = sol_stream_printf(state, stream, "%s", str->str);
+		sol_obj_free(str);
+	}
 	sol_obj_free(obj);
-	sol_obj_free(str);
 	sol_obj_free(stream);
 	return sol_new_int(state, sz);
 }
@@ -2682,8 +3013,8 @@ sol_object_t *sol_f_stream_read_buffer(sol_state_t *state, sol_object_t *args) {
 	sol_object_t *stream = sol_list_get_index(state, args, 0), *amt = sol_list_get_index(state, args, 1), *iamt, *res;
 	char *s = NULL, *p;
 	size_t count = 0, max = 0, pos, end;
-	if(sol_is_string(amt)) {
-		if(sol_string_eq(state, amt, "ALL")) {
+	if(sol_is_name(amt)) {
+		if(sol_name_eq(state, amt, "ALL")) {
 			pos = sol_stream_ftell(state, stream);
 			sol_stream_fseek(state, stream, 0, SEEK_END);
 			end = sol_stream_ftell(state, stream);
@@ -2697,7 +3028,7 @@ sol_object_t *sol_f_stream_read_buffer(sol_state_t *state, sol_object_t *args) {
 				return sol_set_error_string(state, "IO read error");
 			}
 			count = end - pos;
-		} else if(sol_string_eq(state, amt, "LINE")) {
+		} else if(sol_name_eq(state, amt, "LINE")) {
 			s = malloc(STDIO_CHUNK_SIZE * sizeof(char));
 			sol_stream_fgets(state, stream, s, STDIO_CHUNK_SIZE);
 			count = strlen(s);
@@ -2720,12 +3051,14 @@ sol_object_t *sol_f_stream_read_buffer(sol_state_t *state, sol_object_t *args) {
 	return res;
 }
 
+/*
 sol_object_t *sol_f_stream_read(sol_state_t *state, sol_object_t *args) {
 	sol_object_t *buf = sol_f_stream_read_buffer(state, args);
 	sol_object_t *str = sol_new_string(state, buf->buffer);
 	sol_obj_free(buf);
 	return str;
 }
+*/
 
 sol_object_t *sol_f_stream_seek(sol_state_t *state, sol_object_t *args) {
 	sol_object_t *stream = sol_list_get_index(state, args, 0), *offset = sol_list_get_index(state, args, 1), *whence = sol_list_get_index(state, args, 2);

lex.yy.c

@@ -8,96 +8,34 @@
 #define FLEX_SCANNER
 #define YY_FLEX_MAJOR_VERSION 2
 #define YY_FLEX_MINOR_VERSION 6
-#define YY_FLEX_SUBMINOR_VERSION 3
+#define YY_FLEX_SUBMINOR_VERSION 4
 #if YY_FLEX_SUBMINOR_VERSION > 0
 #define FLEX_BETA
 #endif
 
-    #define yy_create_buffer yy_create_buffer
-
-    #define yy_delete_buffer yy_delete_buffer
-
-    #define yy_scan_buffer yy_scan_buffer
-
-    #define yy_scan_string yy_scan_string
-
-    #define yy_scan_bytes yy_scan_bytes
-
-    #define yy_init_buffer yy_init_buffer
-
-    #define yy_flush_buffer yy_flush_buffer
-
-    #define yy_load_buffer_state yy_load_buffer_state
-
-    #define yy_switch_to_buffer yy_switch_to_buffer
-
-    #define yypush_buffer_state yypush_buffer_state
-
-    #define yypop_buffer_state yypop_buffer_state
-
-    #define yyensure_buffer_stack yyensure_buffer_stack
-
-    #define yylex yylex
-
-    #define yyrestart yyrestart
-
-    #define yylex_init yylex_init
-
-    #define yylex_init_extra yylex_init_extra
-
-    #define yylex_destroy yylex_destroy
-
-    #define yyget_debug yyget_debug
-
-    #define yyset_debug yyset_debug
-
-    #define yyget_extra yyget_extra
-
-    #define yyset_extra yyset_extra
-
-    #define yyget_in yyget_in
-
-    #define yyset_in yyset_in
-
-    #define yyget_out yyget_out
-
-    #define yyset_out yyset_out
-
-    #define yyget_leng yyget_leng
-
-    #define yyget_text yyget_text
-
-    #define yyget_lineno yyget_lineno
-
-    #define yyset_lineno yyset_lineno
-
-    #define yywrap yywrap
-
-    #define yyget_lval yyget_lval
-
-    #define yyset_lval yyset_lval
-
-    #define yyget_lloc yyget_lloc
-
-    #define yyset_lloc yyset_lloc
-
-    #define yyalloc yyalloc
-
-    #define yyrealloc yyrealloc
-
-    #define yyfree yyfree
-
-    #define yytext yytext
-
-    #define yyleng yyleng
-
-    #define yyin yyin
+#ifdef yyget_lval
+#define yyget_lval_ALREADY_DEFINED
+#else
+#define yyget_lval yyget_lval
+#endif
 
-    #define yyout yyout
+#ifdef yyset_lval
+#define yyset_lval_ALREADY_DEFINED
+#else
+#define yyset_lval yyset_lval
+#endif
 
-    #define yy_flex_debug yy_flex_debug
+#ifdef yyget_lloc
+#define yyget_lloc_ALREADY_DEFINED
+#else
+#define yyget_lloc yyget_lloc
+#endif
 
-    #define yylineno yylineno
+#ifdef yyset_lloc
+#define yyset_lloc_ALREADY_DEFINED
+#else
+#define yyset_lloc yyset_lloc
+#endif
 
 /* First, we deal with  platform-specific or compiler-specific issues. */
 
@@ -169,10 +107,16 @@ typedef unsigned int flex_uint32_t;
 #define UINT32_MAX             (4294967295U)
 #endif
 
+#ifndef SIZE_MAX
+#define SIZE_MAX               (~(size_t)0)
+#endif
+
 #endif /* ! C99 */
 
 #endif /* ! FLEXINT_H */
 
+/* begin standard C++ headers. */
+
 /* TODO: this is always defined, so inline it */
 #define yyconst const
 
@@ -204,7 +148,7 @@ typedef unsigned int flex_uint32_t;
 /* Action number for EOF rule of a given start state. */
 #define YY_STATE_EOF(state) (YY_END_OF_BUFFER + state + 1)
 /* Special action meaning "start processing a new file". */
-#define YY_NEW_FILE yyrestart(yyin  )
+#define YY_NEW_FILE yyrestart( yyin  )
 #define YY_END_OF_BUFFER_CHAR 0
 
 /* Size of default input buffer. */
@@ -370,7 +314,7 @@ void yypop_buffer_state ( void );
 static void yyensure_buffer_stack ( void );
 static void yy_load_buffer_state ( void );
 static void yy_init_buffer ( YY_BUFFER_STATE b, FILE *file  );
-#define YY_FLUSH_BUFFER yy_flush_buffer(YY_CURRENT_BUFFER )
+#define YY_FLUSH_BUFFER yy_flush_buffer( YY_CURRENT_BUFFER )
 
 YY_BUFFER_STATE yy_scan_buffer ( char *base, yy_size_t size  );
 YY_BUFFER_STATE yy_scan_string ( const char *yy_str  );
@@ -386,7 +330,7 @@ void yyfree ( void *  );
 	if ( ! YY_CURRENT_BUFFER ){ \
         yyensure_buffer_stack (); \
 		YY_CURRENT_BUFFER_LVALUE =    \
-            yy_create_buffer(yyin,YY_BUF_SIZE ); \
+            yy_create_buffer( yyin, YY_BUF_SIZE ); \
 	} \
 	YY_CURRENT_BUFFER_LVALUE->yy_is_interactive = is_interactive; \
 	}
@@ -395,7 +339,7 @@ void yyfree ( void *  );
 	if ( ! YY_CURRENT_BUFFER ){\
         yyensure_buffer_stack (); \
 		YY_CURRENT_BUFFER_LVALUE =    \
-            yy_create_buffer(yyin,YY_BUF_SIZE ); \
+            yy_create_buffer( yyin, YY_BUF_SIZE ); \
 	} \
 	YY_CURRENT_BUFFER_LVALUE->yy_at_bol = at_bol; \
 	}
@@ -677,7 +621,7 @@ static void update_loc(YYLTYPE *yylloc, char *yytext){
 
 #define YY_USER_ACTION update_loc(yylloc, yytext);
 
-#line 680 "lex.yy.c"
+#line 624 "lex.yy.c"
 /* This is the right way to do it, but it keeps generating token $undefined.
 
 %x STRING
@@ -693,7 +637,7 @@ static void update_loc(YYLTYPE *yylloc, char *yytext){
 <STRING>. { str_putc(*yytext); }
 
 */
-#line 696 "lex.yy.c"
+#line 640 "lex.yy.c"
 
 #define INITIAL 0
 
@@ -924,17 +868,17 @@ YY_DECL
 		if ( ! YY_CURRENT_BUFFER ) {
 			yyensure_buffer_stack ();
 			YY_CURRENT_BUFFER_LVALUE =
-				yy_create_buffer(yyin,YY_BUF_SIZE );
+				yy_create_buffer( yyin, YY_BUF_SIZE );
 		}
 
-		yy_load_buffer_state( );
+		yy_load_buffer_state(  );
 		}
 
 	{
 #line 85 "tokenizer.lex"
 
 
-#line 937 "lex.yy.c"
+#line 881 "lex.yy.c"
 
 	while ( /*CONSTCOND*/1 )		/* loops until end-of-file is reached */
 		{
@@ -1004,369 +948,366 @@ YY_RULE_SETUP
 	YY_BREAK
 case 3:
 /* rule 3 can match eol */
-YY_RULE_SETUP
-#line 91 "tokenizer.lex"
-{ *yylval = strdup(yytext+1); ((char *) *yylval)[yyleng-2] = 0; return STRING; }
-	YY_BREAK
+#line 92 "tokenizer.lex"
 case 4:
 /* rule 4 can match eol */
 YY_RULE_SETUP
-#line 93 "tokenizer.lex"
-{ *yylval = strdup(yytext+1); ((char *) *yylval)[yyleng-2] = 0; return STRING; }
+#line 92 "tokenizer.lex"
+{ *yylval = malloc(sizeof(unsigned long) + (yyleng - 2) * sizeof(char)); *((unsigned long *) *yylval) = yyleng - 2; memcpy(((char *) *yylval) + sizeof(unsigned long), yytext + 1, yyleng - 2); return STRING; }
 	YY_BREAK
 case 5:
 YY_RULE_SETUP
-#line 95 "tokenizer.lex"
+#line 94 "tokenizer.lex"
 { return IF; }
 	YY_BREAK
 case 6:
 YY_RULE_SETUP
-#line 97 "tokenizer.lex"
+#line 96 "tokenizer.lex"
 { return THEN; }
 	YY_BREAK
 case 7:
 YY_RULE_SETUP
-#line 99 "tokenizer.lex"
+#line 98 "tokenizer.lex"
 { return ELSE; }
 	YY_BREAK
 case 8:
 YY_RULE_SETUP
-#line 101 "tokenizer.lex"
+#line 100 "tokenizer.lex"
 { return ELSEIF; }
 	YY_BREAK
 case 9:
 YY_RULE_SETUP
-#line 103 "tokenizer.lex"
+#line 102 "tokenizer.lex"
 { return WHILE; }
 	YY_BREAK
 case 10:
 YY_RULE_SETUP
-#line 105 "tokenizer.lex"
+#line 104 "tokenizer.lex"
 { return FOR; }
 	YY_BREAK
 case 11:
 YY_RULE_SETUP
-#line 107 "tokenizer.lex"
+#line 106 "tokenizer.lex"
 { return IN; }
 	YY_BREAK
 case 12:
 YY_RULE_SETUP
-#line 109 "tokenizer.lex"
+#line 108 "tokenizer.lex"
 { return DO; }
 	YY_BREAK
 case 13:
 YY_RULE_SETUP
-#line 111 "tokenizer.lex"
+#line 110 "tokenizer.lex"
 { return FUNC; }
 	YY_BREAK
 case 14:
 YY_RULE_SETUP
-#line 113 "tokenizer.lex"
+#line 112 "tokenizer.lex"
 { return LAMBDA; }
 	YY_BREAK
 case 15:
 YY_RULE_SETUP
-#line 115 "tokenizer.lex"
+#line 114 "tokenizer.lex"
 { return RETURN; }
 	YY_BREAK
 case 16:
 YY_RULE_SETUP
-#line 117 "tokenizer.lex"
+#line 116 "tokenizer.lex"
 { return BREAK; }
 	YY_BREAK
 case 17:
 YY_RULE_SETUP
-#line 119 "tokenizer.lex"
+#line 118 "tokenizer.lex"
 { return CONTINUE; }
 	YY_BREAK
 case 18:
 YY_RULE_SETUP
-#line 121 "tokenizer.lex"
+#line 120 "tokenizer.lex"
 { return END; }
 	YY_BREAK
 case 19:
 YY_RULE_SETUP
-#line 123 "tokenizer.lex"
+#line 122 "tokenizer.lex"
 { return NONE; }
 	YY_BREAK
 case 20:
 YY_RULE_SETUP
-#line 125 "tokenizer.lex"
+#line 124 "tokenizer.lex"
 { return PLUS; }
 	YY_BREAK
 case 21:
 YY_RULE_SETUP
-#line 127 "tokenizer.lex"
+#line 126 "tokenizer.lex"
 { return MINUS; }
 	YY_BREAK
 case 22:
 YY_RULE_SETUP
-#line 129 "tokenizer.lex"
+#line 128 "tokenizer.lex"
 { return STAR; }
 	YY_BREAK
 case 23:
 YY_RULE_SETUP
-#line 131 "tokenizer.lex"
+#line 130 "tokenizer.lex"
 { return SLASH; }
 	YY_BREAK
 case 24:
 YY_RULE_SETUP
-#line 133 "tokenizer.lex"
+#line 132 "tokenizer.lex"
 { return PERCENT; }
 	YY_BREAK
 case 25:
 YY_RULE_SETUP
-#line 135 "tokenizer.lex"
+#line 134 "tokenizer.lex"
 { return PERCENT; }
 	YY_BREAK
 case 26:
 YY_RULE_SETUP
-#line 137 "tokenizer.lex"
+#line 136 "tokenizer.lex"
 { return DSTAR; }
 	YY_BREAK
 case 27:
 YY_RULE_SETUP
-#line 139 "tokenizer.lex"
+#line 138 "tokenizer.lex"
 { return BAND; }
 	YY_BREAK
 case 28:
 YY_RULE_SETUP
-#line 141 "tokenizer.lex"
+#line 140 "tokenizer.lex"
 { return BOR; }
 	YY_BREAK
 case 29:
 YY_RULE_SETUP
-#line 143 "tokenizer.lex"
+#line 142 "tokenizer.lex"
 { return BXOR; }
 	YY_BREAK
 case 30:
 YY_RULE_SETUP
-#line 145 "tokenizer.lex"
+#line 144 "tokenizer.lex"
 { return BNOT; }
 	YY_BREAK
 case 31:
 YY_RULE_SETUP
-#line 147 "tokenizer.lex"
+#line 146 "tokenizer.lex"
 { return LAND; }
 	YY_BREAK
 case 32:
 YY_RULE_SETUP
-#line 149 "tokenizer.lex"
+#line 148 "tokenizer.lex"
 { return LAND; }
 	YY_BREAK
 case 33:
 YY_RULE_SETUP
-#line 151 "tokenizer.lex"
+#line 150 "tokenizer.lex"
 { return LOR; }
 	YY_BREAK
 case 34:
 YY_RULE_SETUP
-#line 153 "tokenizer.lex"
+#line 152 "tokenizer.lex"
 { return LOR; }
 	YY_BREAK
 case 35:
 YY_RULE_SETUP
-#line 155 "tokenizer.lex"
+#line 154 "tokenizer.lex"
 { return LNOT; }
 	YY_BREAK
 case 36:
 YY_RULE_SETUP
-#line 157 "tokenizer.lex"
+#line 156 "tokenizer.lex"
 { return LNOT; }
 	YY_BREAK
 case 37:
 YY_RULE_SETUP
-#line 159 "tokenizer.lex"
+#line 158 "tokenizer.lex"
 { *yylval = malloc(sizeof(long)); *((long *) *yylval) = 1; return INT;  }
 	YY_BREAK
 case 38:
 YY_RULE_SETUP
-#line 161 "tokenizer.lex"
+#line 160 "tokenizer.lex"
 { *yylval = malloc(sizeof(long)); *((long *) *yylval) = 1; return INT;  }
 	YY_BREAK
 case 39:
 YY_RULE_SETUP
-#line 163 "tokenizer.lex"
+#line 162 "tokenizer.lex"
 { *yylval = malloc(sizeof(long)); *((long *) *yylval) = 0; return INT;  }
 	YY_BREAK
 case 40:
 YY_RULE_SETUP
-#line 165 "tokenizer.lex"
+#line 164 "tokenizer.lex"
 { *yylval = malloc(sizeof(long)); *((long *) *yylval) = 0; return INT;  }
 	YY_BREAK
 case 41:
 YY_RULE_SETUP
-#line 167 "tokenizer.lex"
+#line 166 "tokenizer.lex"
 { return ASSIGN; }
 	YY_BREAK
 case 42:
 YY_RULE_SETUP
-#line 169 "tokenizer.lex"
+#line 168 "tokenizer.lex"
 { return ASSIGNPLUS; }
 	YY_BREAK
 case 43:
 YY_RULE_SETUP
-#line 171 "tokenizer.lex"
+#line 170 "tokenizer.lex"
 { return ASSIGNMINUS; }
 	YY_BREAK
 case 44:
 YY_RULE_SETUP
-#line 173 "tokenizer.lex"
+#line 172 "tokenizer.lex"
 { return ASSIGNSTAR; }
 	YY_BREAK
 case 45:
 YY_RULE_SETUP
-#line 175 "tokenizer.lex"
+#line 174 "tokenizer.lex"
 { return ASSIGNSLASH; }
 	YY_BREAK
 case 46:
 YY_RULE_SETUP
-#line 177 "tokenizer.lex"
+#line 176 "tokenizer.lex"
 { return ASSIGNDSTAR; }
 	YY_BREAK
 case 47:
 YY_RULE_SETUP
-#line 179 "tokenizer.lex"
+#line 178 "tokenizer.lex"
 { return ASSIGNBAND; }
 	YY_BREAK
 case 48:
 YY_RULE_SETUP
-#line 181 "tokenizer.lex"
+#line 180 "tokenizer.lex"
 { return ASSIGNBOR; }
 	YY_BREAK
 case 49:
 YY_RULE_SETUP
-#line 183 "tokenizer.lex"
+#line 182 "tokenizer.lex"
 { return ASSIGNBXOR; }
 	YY_BREAK
 case 50:
 YY_RULE_SETUP
-#line 185 "tokenizer.lex"
+#line 184 "tokenizer.lex"
 { return EQUAL; }
 	YY_BREAK
 case 51:
 YY_RULE_SETUP
-#line 187 "tokenizer.lex"
+#line 186 "tokenizer.lex"
 { return NEQUAL; }
 	YY_BREAK
 case 52:
 YY_RULE_SETUP
-#line 189 "tokenizer.lex"
+#line 188 "tokenizer.lex"
 { return LESS; }
 	YY_BREAK
 case 53:
 YY_RULE_SETUP
-#line 191 "tokenizer.lex"
+#line 190 "tokenizer.lex"
 { return GREATER; }
 	YY_BREAK
 case 54:
 YY_RULE_SETUP
-#line 193 "tokenizer.lex"
+#line 192 "tokenizer.lex"
 { return LESSEQ; }
 	YY_BREAK
 case 55:
 YY_RULE_SETUP
-#line 195 "tokenizer.lex"
+#line 194 "tokenizer.lex"
 { return GREATEREQ; }
 	YY_BREAK
 case 56:
 YY_RULE_SETUP
-#line 197 "tokenizer.lex"
+#line 196 "tokenizer.lex"
 { return RSHIFT; }
 	YY_BREAK
 case 57:
 YY_RULE_SETUP
-#line 199 "tokenizer.lex"
+#line 198 "tokenizer.lex"
 { return LSHIFT; }
 	YY_BREAK
 case 58:
 YY_RULE_SETUP
-#line 201 "tokenizer.lex"
+#line 200 "tokenizer.lex"
 { return LBRACE; }
 	YY_BREAK
 case 59:
 YY_RULE_SETUP
-#line 203 "tokenizer.lex"
+#line 202 "tokenizer.lex"
 { return RBRACE; }
 	YY_BREAK
 case 60:
 YY_RULE_SETUP
-#line 205 "tokenizer.lex"
+#line 204 "tokenizer.lex"
 { return LBRACKET; }
 	YY_BREAK
 case 61:
 YY_RULE_SETUP
-#line 207 "tokenizer.lex"
+#line 206 "tokenizer.lex"
 { return RBRACKET; }
 	YY_BREAK
 case 62:
 YY_RULE_SETUP
-#line 209 "tokenizer.lex"
+#line 208 "tokenizer.lex"
 { return BLPAREN; } /* "Breaking" paren, not allowed to introduce a call_expr */
 	YY_BREAK
 case 63:
 YY_RULE_SETUP
-#line 211 "tokenizer.lex"
+#line 210 "tokenizer.lex"
 { return LPAREN; } 
 	YY_BREAK
 case 64:
 YY_RULE_SETUP
-#line 213 "tokenizer.lex"
+#line 212 "tokenizer.lex"
 { return RPAREN; }
 	YY_BREAK
 case 65:
 YY_RULE_SETUP
-#line 215 "tokenizer.lex"
+#line 214 "tokenizer.lex"
 { return DOT; }
 	YY_BREAK
 case 66:
 YY_RULE_SETUP
-#line 217 "tokenizer.lex"
+#line 216 "tokenizer.lex"
 { return COLON; }
 	YY_BREAK
 case 67:
 YY_RULE_SETUP
-#line 219 "tokenizer.lex"
+#line 218 "tokenizer.lex"
 { return SEMICOLON; }
 	YY_BREAK
 case 68:
 YY_RULE_SETUP
-#line 221 "tokenizer.lex"
+#line 220 "tokenizer.lex"
 { return COMMA; }
 	YY_BREAK
 case 69:
 YY_RULE_SETUP
-#line 223 "tokenizer.lex"
+#line 222 "tokenizer.lex"
 { return POUND; }
 	YY_BREAK
 case 70:
 YY_RULE_SETUP
-#line 225 "tokenizer.lex"
+#line 224 "tokenizer.lex"
 { return TBANG; }
 	YY_BREAK
 case 71:
 YY_RULE_SETUP
-#line 227 "tokenizer.lex"
+#line 226 "tokenizer.lex"
 { *yylval = (void *) strdup(yytext); return IDENT; }
 	YY_BREAK
 case 72:
 /* rule 72 can match eol */
 YY_RULE_SETUP
-#line 229 "tokenizer.lex"
+#line 228 "tokenizer.lex"
 /* Skip comments */
 	YY_BREAK
 case 73:
 /* rule 73 can match eol */
 YY_RULE_SETUP
-#line 231 "tokenizer.lex"
+#line 230 "tokenizer.lex"
 /* Skip whitespace */
 	YY_BREAK
 case 74:
 YY_RULE_SETUP
-#line 233 "tokenizer.lex"
+#line 232 "tokenizer.lex"
 ECHO;
 	YY_BREAK
-#line 1369 "lex.yy.c"
+#line 1310 "lex.yy.c"
 case YY_STATE_EOF(INITIAL):
 	yyterminate();
 
@@ -1444,7 +1385,7 @@ case YY_STATE_EOF(INITIAL):
 				{
 				(yy_did_buffer_switch_on_eof) = 0;
 
-				if ( yywrap( ) )
+				if ( yywrap(  ) )
 					{
 					/* Note: because we've taken care in
 					 * yy_get_next_buffer() to have set up
@@ -1576,7 +1517,8 @@ static int yy_get_next_buffer (void)
 
 				b->yy_ch_buf = (char *)
 					/* Include room in for 2 EOB chars. */
-					yyrealloc((void *) b->yy_ch_buf,(yy_size_t) (b->yy_buf_size + 2)  );
+					yyrealloc( (void *) b->yy_ch_buf,
+							 (yy_size_t) (b->yy_buf_size + 2)  );
 				}
 			else
 				/* Can't grow it, we don't own it. */
@@ -1608,7 +1550,7 @@ static int yy_get_next_buffer (void)
 		if ( number_to_move == YY_MORE_ADJ )
 			{
 			ret_val = EOB_ACT_END_OF_FILE;
-			yyrestart(yyin  );
+			yyrestart( yyin  );
 			}
 
 		else
@@ -1625,9 +1567,12 @@ static int yy_get_next_buffer (void)
 	if (((yy_n_chars) + number_to_move) > YY_CURRENT_BUFFER_LVALUE->yy_buf_size) {
 		/* Extend the array by 50%, plus the number we really need. */
 		int new_size = (yy_n_chars) + number_to_move + ((yy_n_chars) >> 1);
-		YY_CURRENT_BUFFER_LVALUE->yy_ch_buf = (char *) yyrealloc((void *) YY_CURRENT_BUFFER_LVALUE->yy_ch_buf,(yy_size_t) new_size  );