// old token

// do atoi in an array ar[start] with length len
to aatoi ar start len | i c v =
	v = 0
	for i start start+len-1
		c = ar[i]
		if ! isNum c
			seterror "invalid integer"
		v = v*10 + c - 48
	v

// convert hex to int, similar to aatoi above
to ahtoi ar start len | i c v h =
	h = 0
	v = 0
	for i start start+len-1
		c = ar[i]
		if ! isHex c
			seterror "invalid hex number"
		v = hexval c		// c is valid hex
		if (i == start) & (v > 7)
			h = v - 16		// sign extension
		else
			h = h*16 + v
	h

// return pointer to begin of token and CLEN
// if "//" is found set CLEN = 0
to token | pos a c =
	pos = TP
	if (CLEN == 0) | (inbuf[pos] == 0)	// end buffer
		fgets FI inbuf			// get a line
//		nl
		line = line + 1
		pos = 0
	while isSpace inbuf[pos]	// skip space
		pos = pos + 1
	tokcol = pos
	c = inbuf[pos]
	if isSep c
		CLEN = 1
		a = inbuf[pos+1]		// lookahead
		if (c==47) & (a==47) 	// "//" comment
			CLEN = 0			// denote blank line
		if c == 0
			CLEN = 0
	else
		a = pos
		while isLetter inbuf[a]
			a = a + 1
		CLEN = a - pos
	pos

// affect CLEN, tok, tokvalue, tokstring. change TP
to lex1 | c d =
	TP = token
	while CLEN == 0		// read until get a token
		TP = token
	c = inbuf[TP]
	if c == 255			// EOF
		tok = tkEOF
		break
	if isNum c
		tokvalue = aatoi inbuf TP CLEN
		tok = tkNUMBER
		TP = TP + CLEN
		break

	d = inbuf[TP+1]	// lookahead
	case c
		34:					// " string
			collectstring
			strpack tokstring inbuf TP+1 CLEN-2
			tok = tkSTRING
		35:					// # hex num
			tokvalue = ahtoi inbuf TP+1 CLEN-1
			tok = tkNUMBER
		33: if d == 61		// !=
				CLEN = 2
				tok = tkNE
			else
				tok = tkNOT
		37: tok = tkMOD
		38: tok = tkAND
		40: tok = tkLPAREN
		41: tok = tkRPAREN
		42: tok = tkSTAR
		43: tok = tkPLUS
		45: tok = tkMINUS
		47: tok = tkSLASH
		58: tok = tkCOLON
		60: if d == 60		// <<
				CLEN = 2
				tok = tkLTLT
			else if d == 61 // <=
				CLEN = 2
				tok = tkLE
			else
				tok = tkLT
		61: if d == 61		// ==
				CLEN = 2
				tok = tkEQEQ
			else
				tok = tkEQ
		62: if d == 62		// >>
				CLEN = 2
				tok = tkGTGT
			else if d == 61 // >=
				CLEN = 2
				tok = tkGE
			else
				tok = tkGT
		91: tok = tkLBRACKET // [
		93: tok = tkRBRACKET // ]
		94: tok = tkCARET	 // ^
		123: tok = tkBB		 // {
		124: tok = tkBAR	 // |
		125: tok = tkBE		 // }

		else:		// it is an indentifier
			strpack tokstring inbuf TP CLEN
			tokvalue = install tokstring
			tok = tkIDEN
			if (getType tokvalue) == tyKEY // key word
				tok = getRef tokvalue
	// end case
	TP = TP + CLEN
