[nasm:master] preproc: unify token codes between the preprocessor and assembler

nasm-bot for H. Peter Anvin (Intel) hpa at zytor.com
Tue Jul 14 11:54:05 PDT 2020


Commit-ID:  20e0d616dc954d567c8bf2c7e11cc5d6c10ac544
Gitweb:     http://repo.or.cz/w/nasm.git?a=commitdiff;h=20e0d616dc954d567c8bf2c7e11cc5d6c10ac544
Author:     H. Peter Anvin (Intel) <hpa at zytor.com>
AuthorDate: Tue, 14 Jul 2020 11:46:05 -0700
Committer:  H. Peter Anvin (Intel) <hpa at zytor.com>
CommitDate: Tue, 14 Jul 2020 11:50:55 -0700

preproc: unify token codes between the preprocessor and assembler

The different token codes between the preprocessor and the assembler
is a completely unnecessary headache. Furthermore, lumping all the
operators under TOK_OTHER in the preprocessor causes a whole bunch of
unnecessary headaches.

In combining them, the only tricky part is that PP_CONCAT_MASK() is no
longer usable, as the range of token codes is too large. Replace with
dedicated category masks.

Signed-off-by: H. Peter Anvin (Intel) <hpa at zytor.com>


---
 asm/nasm.c     |   2 +-
 asm/preproc.c  | 757 ++++++++++++++++++++++++++++-----------------------------
 include/nasm.h | 102 ++++++--
 3 files changed, 459 insertions(+), 402 deletions(-)

diff --git a/asm/nasm.c b/asm/nasm.c
index 8b8982f2..5155f35b 100644
--- a/asm/nasm.c
+++ b/asm/nasm.c
@@ -1314,7 +1314,7 @@ static bool process_arg(char *p, char *q, int pass)
                     break;
                 case OPT_PRAGMA:
                     if (pass == 2)
-                        pp_pre_command("pragma", param);
+                        pp_pre_command("%pragma", param);
                     break;
                 case OPT_BEFORE:
                     if (pass == 2)
diff --git a/asm/preproc.c b/asm/preproc.c
index 0dfde9a5..e8e93341 100644
--- a/asm/preproc.c
+++ b/asm/preproc.c
@@ -122,51 +122,27 @@ typedef struct Cond Cond;
  * tok_smac_param(0) but the one representing `y' will be
  * tok_smac_param(1); see the accessor functions below.
  *
- * TOK_INTERNAL_STRING is a string which has been unquoted, but should
+ * TOK_INTERNAL_STR is a string which has been unquoted, but should
  * be treated as if it was a quoted string. The code is free to change
- * one into the other at will. TOK_NAKED_STRING is a text token which
+ * one into the other at will. TOK_NAKED_STR is a text token which
  * should be treated as a string, but which MUST NOT be turned into a
- * quoted string. TOK_INTERNAL_STRINGs can contain any character,
- * including NUL, but TOK_NAKED_STRING must be a valid C string.
+ * quoted string. TOK_INTERNAL_STRs can contain any character,
+ * including NUL, but TOK_NAKED_STR must be a valid C string.
  */
-enum pp_token_type {
-    TOK_NONE = 0, TOK_WHITESPACE, TOK_COMMENT,
-    TOK_CORRUPT,		/* Token text modified in an unsafe manner, now bogus */
-    TOK_BLOCK,			/* Storage block pointer, not a real token */
-    TOK_ID,
-    TOK_PREPROC_ID, TOK_MMACRO_PARAM, TOK_LOCAL_SYMBOL,
-    TOK_LOCAL_MACRO, TOK_ENVIRON, TOK_STRING,
-    TOK_NUMBER, TOK_FLOAT, TOK_OTHER,
-    TOK_INTERNAL_STRING, TOK_NAKED_STRING,
-    TOK_PREPROC_Q, TOK_PREPROC_QQ,
-    TOK_PASTE,              /* %+ */
-    TOK_COND_COMMA,         /* %, */
-    TOK_INDIRECT,           /* %[...] */
-    TOK_XDEF_PARAM,         /* Used during %xdefine processing */
-    TOK_SMAC_START_PARAMS,  /* MUST BE LAST IN THE LIST!!! */
-    TOK_MAX = INT_MAX       /* Keep compiler from reducing the range */
-};
 
-static inline enum pp_token_type tok_smac_param(int param)
+static inline enum token_type tok_smac_param(int param)
 {
-    return TOK_SMAC_START_PARAMS + param;
+    return TOKEN_SMAC_START_PARAMS + param;
 }
-static int smac_nparam(enum pp_token_type toktype)
+static int smac_nparam(enum token_type toktype)
 {
-    return toktype - TOK_SMAC_START_PARAMS;
+    return toktype - TOKEN_SMAC_START_PARAMS;
 }
-static bool is_smac_param(enum pp_token_type toktype)
+static bool is_smac_param(enum token_type toktype)
 {
-    return toktype >= TOK_SMAC_START_PARAMS;
+    return toktype >= TOKEN_SMAC_START_PARAMS;
 }
 
-#define PP_CONCAT_MASK(x) (1U << (x))
-
-struct tokseq_match {
-    int mask_head;
-    int mask_tail;
-};
-
 /*
  * This is tuned so struct Token should be 64 bytes on 64-bit
  * systems and 32 bytes on 32-bit systems. It enables them
@@ -178,17 +154,17 @@ struct tokseq_match {
  * if the length is passed through an interface with type "int",
  * and is absurdly large anyway.
  *
- * For the text mode, in pointer mode the pointer is stored at the end
- * of the union and the pad field is cleared. This allows short tokens
- * to be unconditionally tested for by only looking at the first text
- * bytes and not examining the type or len fields.
+ * Earlier versions of the source code incorrectly stated that
+ * examining the text string alone can be unconditionally valid. This
+ * is incorrect, as some token types strip parts of the string,
+ * e.g. indirect tokens.
  */
-#define INLINE_TEXT (7*sizeof(char *)-sizeof(enum pp_token_type)-sizeof(unsigned int)-1)
+#define INLINE_TEXT (7*sizeof(char *)-sizeof(enum token_type)-sizeof(unsigned int)-1)
 #define MAX_TEXT (INT_MAX-2)
 
 struct Token {
     Token *next;
-    enum pp_token_type type;
+    enum token_type type;
     unsigned int len;
     union {
         char a[INLINE_TEXT+1];
@@ -658,9 +634,9 @@ static Token *make_tok_num(Token *next, int64_t val);
 static Token *make_tok_qstr(Token *next, const char *str);
 static Token *make_tok_qstr_len(Token *next, const char *str, size_t len);
 static Token *make_tok_char(Token *next, char op);
-static Token *new_Token(Token * next, enum pp_token_type type,
+static Token *new_Token(Token * next, enum token_type type,
                         const char *text, size_t txtlen);
-static Token *new_Token_free(Token * next, enum pp_token_type type,
+static Token *new_Token_free(Token * next, enum token_type type,
                              char *text, size_t txtlen);
 static Token *dup_Token(Token *next, const Token *src);
 static Token *new_White(Token *next);
@@ -669,18 +645,29 @@ static Token *steal_Token(Token *dst, Token *src);
 static const struct use_package *
 get_use_pkg(Token *t, const char *dname, const char **name);
 static void mark_smac_params(Token *tline, const SMacro *tmpl,
-                             enum pp_token_type type);
+                             enum token_type type);
+
+/* Safe extraction of token type */
+static inline enum token_type tok_type(const Token *x)
+{
+    return x ? x->type : TOKEN_EOS;
+}
 
 /* Safe test for token type, false on x == NULL */
-static inline bool tok_type(const Token *x, enum pp_token_type t)
+static inline bool tok_is(const Token *x, enum token_type t)
+{
+    return tok_type(x) == t;
+}
+/* True if token is any other kind other that "c", but not NULL */
+static inline bool tok_isnt(const Token *x, enum token_type t)
 {
-    return x && x->type == t;
+    return x && x->type != t;
 }
 
 /* Whitespace token? */
 static inline bool tok_white(const Token *x)
 {
-    return tok_type(x, TOK_WHITESPACE);
+    return tok_is(x, TOKEN_WHITESPACE);
 }
 
 /* Skip past any whitespace */
@@ -701,25 +688,9 @@ static Token *zap_white(Token *x)
     return x;
 }
 
-/*
- * Single special character tests. The use of & rather than && is intentional; it
- * tells the compiler that it is safe to access text.a[1] unconditionally; hopefully
- * a smart compiler should turn it into a 16-bit memory reference.
- */
-static inline bool tok_is(const Token *x, char c)
-{
-    return x && ((x->text.a[0] == c) & !x->text.a[1]);
-}
-
-/* True if any other kind of token that "c", but not NULL */
-static inline bool tok_isnt(const Token *x, char c)
-{
-    return x && !((x->text.a[0] == c) & !x->text.a[1]);
-}
-
 /*
  * Unquote a token if it is a string, and set its type to
- * TOK_INTERNAL_STRING.
+ * TOKEN_INTERNAL_STR.
  */
 
 /*
@@ -731,13 +702,13 @@ static const char *unquote_token_anystr(Token *t, uint32_t badctl, char qstart)
     size_t nlen, olen;
     char *p;
 
-    if (t->type != TOK_STRING)
+    if (t->type != TOKEN_STR)
 	return tok_text(t);
 
     olen = t->len;
     p = (olen > INLINE_TEXT) ? t->text.p.ptr : t->text.a;
     t->len = nlen = nasm_unquote_anystr(p, NULL, badctl, qstart);
-    t->type = TOK_INTERNAL_STRING;
+    t->type = TOKEN_INTERNAL_STR;
 
     if (olen <= INLINE_TEXT || nlen > INLINE_TEXT)
         return p;
@@ -764,14 +735,14 @@ static const char *unquote_token_cstr(Token *t)
 }
 
 /*
- * Convert a TOK_INTERNAL_STRING token to a quoted
- * TOK_STRING tokens.
+ * Convert a TOKEN_INTERNAL_STR token to a quoted
+ * TOKEN_STR tokens.
  */
 static Token *quote_any_token(Token *t);
 static inline unused_func
 Token *quote_token(Token *t)
 {
-    if (likely(!tok_is(t, TOK_INTERNAL_STRING)))
+    if (likely(!tok_is(t, TOKEN_INTERNAL_STR)))
 	return t;
 
     return quote_any_token(t);
@@ -779,7 +750,7 @@ Token *quote_token(Token *t)
 
 /*
  * Convert *any* kind of token to a quoted
- * TOK_STRING token.
+ * TOKEN_STR token.
  */
 static Token *quote_any_token(Token *t)
 {
@@ -787,7 +758,7 @@ static Token *quote_any_token(Token *t)
     char *p;
 
     p = nasm_quote(tok_text(t), &len);
-    t->type = TOK_STRING;
+    t->type = TOKEN_STR;
     return set_text_free(t, p, len);
 }
 
@@ -823,18 +794,18 @@ static const char *pp_getenv(const Token *t, bool warn)
 	return NULL;
 
     switch (t->type) {
-    case TOK_ENVIRON:
+    case TOKEN_ENVIRON:
 	txt += 2;		/* Skip leading %! */
 	is_string = nasm_isquote(*txt);
 	break;
 
-    case TOK_STRING:
+    case TOKEN_STR:
 	is_string = true;
 	break;
 
-    case TOK_INTERNAL_STRING:
-    case TOK_NAKED_STRING:
-    case TOK_ID:
+    case TOKEN_INTERNAL_STR:
+    case TOKEN_NAKED_STR:
+    case TOKEN_ID:
 	is_string = false;
 	break;
 
@@ -1315,7 +1286,7 @@ static char *read_line(void)
  */
 static Token *tokenize(const char *line)
 {
-    enum pp_token_type type;
+    enum token_type type;
     Token *list = NULL;
     Token *t, **tail = &list;
 
@@ -1411,97 +1382,98 @@ static Token *tokenize(const char *line)
 
             /* Classify here, to handle %{...} correctly */
             if (toklen < 2) {
-                type = TOK_OTHER;   /* % operator */
+                type = '%';     /* % operator */
             } else {
                 char c0 = line[1];
 
                 switch (c0) {
                 case '+':
-                    type = (toklen == 2) ? TOK_PASTE : TOK_MMACRO_PARAM;
+                    type = (toklen == 2) ? TOKEN_PASTE : TOKEN_MMACRO_PARAM;
                     break;
 
                 case '-':
-                    type = TOK_MMACRO_PARAM;
+                    type = TOKEN_MMACRO_PARAM;
                     break;
 
                 case '?':
                     if (toklen == 2)
-                        type = TOK_PREPROC_Q;
+                        type = TOKEN_PREPROC_Q;
                     else if (toklen == 3 && line[2] == '?')
-                        type = TOK_PREPROC_QQ;
+                        type = TOKEN_PREPROC_QQ;
                     else
-                        type = TOK_PREPROC_ID;
+                        type = TOKEN_PREPROC_ID;
                     break;
 
                 case '!':
-                    type = (toklen == 2) ? TOK_OTHER : TOK_ENVIRON;
+                    type = (toklen == 2) ? TOKEN_OTHER : TOKEN_ENVIRON;
                     break;
 
                 case '%':
-                    type = (toklen == 2) ? TOK_OTHER : TOK_LOCAL_SYMBOL;
+                    type = (toklen == 2) ? TOKEN_SMOD : TOKEN_LOCAL_SYMBOL;
                     break;
 
                 case '$':
-                    type = (toklen == 2) ? TOK_OTHER : TOK_LOCAL_MACRO;
+                    type = (toklen == 2) ? TOKEN_OTHER : TOKEN_LOCAL_MACRO;
                     break;
 
                 case '[':
                     line += 2;  /* Skip %[ */
                     firstchar = *line; /* Don't clobber */
                     toklen -= 2;
-                    type = TOK_INDIRECT;
+                    type = TOKEN_INDIRECT;
                     break;
 
                 case ',':
-                    type = (toklen == 2) ? TOK_COND_COMMA : TOK_PREPROC_ID;
+                    type = (toklen == 2) ? TOKEN_COND_COMMA : TOKEN_PREPROC_ID;
                     break;
 
                 case '\'':
                 case '\"':
                 case '`':
                     /* %{'string'} */
-                    type = TOK_PREPROC_ID;
+                    type = TOKEN_PREPROC_ID;
                     break;
 
                 case ':':
-                    type = TOK_MMACRO_PARAM; /* %{:..} */
+                    type = TOKEN_MMACRO_PARAM; /* %{:..} */
                     break;
 
                 default:
                     if (nasm_isdigit(c0))
-                        type = TOK_MMACRO_PARAM;
+                        type = TOKEN_MMACRO_PARAM;
                     else if (nasm_isidchar(c0) || toklen > 2)
-                        type = TOK_PREPROC_ID;
+                        type = TOKEN_PREPROC_ID;
                     else
-                        type = TOK_OTHER;
+                        type = TOKEN_OTHER;
                     break;
                 }
             }
+        } else if (*p == '?' && !nasm_isidchar(p[1])) {
+            /* ? operator */
+            type = TOKEN_QMARK;
         } else if (nasm_isidstart(*p) || (*p == '$' && nasm_isidstart(p[1]))) {
             /*
-             * An identifier. This includes the ? operator, which is
-             * treated as a keyword, not as a special character
-             * operator
+             * A regular identifier. This includes keywords, which are not
+             * special to the preprocessor.
              */
-            type = TOK_ID;
+            type = TOKEN_ID;
             while (nasm_isidchar(*++p))
                 ;
          } else if (nasm_isquote(*p)) {
             /*
              * A string token.
              */
-            type = TOK_STRING;
+            type = TOKEN_STR;
             p = nasm_skip_string(p);
 
             if (*p) {
                 p++;
             } else {
                 nasm_warn(WARN_OTHER, "unterminated string");
-                /* Handling unterminated strings by UNV */
-                /* type = -1; */
+                type = TOKEN_ERRSTR;
             }
         } else if (p[0] == '$' && p[1] == '$') {
-            type = TOK_OTHER;   /* TOKEN_BASE */
+            type = TOKEN_BASE;
             p += 2;
         } else if (nasm_isnumstart(*p)) {
             bool is_hex = false;
@@ -1543,7 +1515,7 @@ static Token *tokenize(const char *line)
                     /*
                      * we need to deal with consequences of the legacy
                      * parser, like "1.nolist" being two tokens
-                     * (TOK_NUMBER, TOK_ID) here; at least give it
+                     * (TOKEN_NUM, TOKEN_ID) here; at least give it
                      * a shot for now.  In the future, we probably need
                      * a flex-based scanner with proper pattern matching
                      * to do it as well as it can be done.  Nothing in
@@ -1567,32 +1539,28 @@ static Token *tokenize(const char *line)
             p--;        /* Point to first character beyond number */
 
             if (p == line+1 && *line == '$') {
-                type = TOK_OTHER; /* TOKEN_HERE */
+                type = TOKEN_HERE;
             } else {
                 if (has_e && !is_hex) {
                     /* 1e13 is floating-point, but 1e13h is not */
                     is_float = true;
                 }
 
-                type = is_float ? TOK_FLOAT : TOK_NUMBER;
+                type = is_float ? TOKEN_FLOAT : TOKEN_NUM;
             }
         } else if (nasm_isspace(*p)) {
-            type = TOK_WHITESPACE;
+            firstchar = ' ';    /* Always a single space */
+            type = TOKEN_WHITESPACE;
             p = nasm_skip_spaces(p);
             /*
              * Whitespace just before end-of-line is discarded by
              * pretending it's a comment; whitespace just before a
              * comment gets lumped into the comment.
              */
-            if (!*p || *p == ';') {
-                type = TOK_COMMENT;
-                while (*p)
-                    p++;
-            }
+            if (!*p || *p == ';')
+                type = TOKEN_EOS;
         } else if (*p == ';') {
-            type = TOK_COMMENT;
-            while (*p)
-                p++;
+            type = TOKEN_EOS;
         } else {
             /*
              * Anything else is an operator of some kind. We check
@@ -1601,14 +1569,18 @@ static Token *tokenize(const char *line)
 	     * character operators (<<<, >>>, <=>) but anything
              * else is a single-character operator.
              */
-            type = TOK_OTHER;
+            type = (unsigned char)*p;
 	    switch (*p++) {
 	    case '>':
 		if (*p == '>') {
 		    p++;
-		    if (*p == '>')
+                    type = TOKEN_SHR;
+		    if (*p == '>') {
+                        type = TOKEN_SAR;
 			p++;
+                    }
 		} else if (*p == '=') {
+                    type = TOKEN_GE;
                     p++;
                 }
 		break;
@@ -1616,30 +1588,58 @@ static Token *tokenize(const char *line)
 	    case '<':
 		if (*p == '<') {
 		    p++;
+                    type = TOKEN_SHR;
 		    if (*p == '<')
 			p++;
 		} else if (*p == '=') {
 		    p++;
-		    if (*p == '>')
+                    type = TOKEN_LE;
+		    if (*p == '>') {
 			p++;
+                        type = TOKEN_LEG;
+                    }
 		} else if (*p == '>') {
 		    p++;
+                    type = TOKEN_NE;
 		}
 		break;
 
 	    case '!':
-		if (*p == '=')
+		if (*p == '=') {
 		    p++;
+                    type = TOKEN_NE;
+                }
 		break;
 
 	    case '/':
+                if (*p == '/') {
+                    p++;
+                    type = TOKEN_SDIV;
+                }
+                break;
 	    case '=':
+                if (*p == '=')
+                    p++;        /* Still TOKEN_EQ == '=' though */
+                break;
 	    case '&':
+                if (*p == '&') {
+                    p++;
+                    type = TOKEN_DBL_AND;
+                }
+                break;
+
 	    case '|':
+                if (*p == '|') {
+                    p++;
+                    type = TOKEN_DBL_OR;
+                }
+                break;
+
 	    case '^':
-		/* These operators can be doubled but nothing else */
-		if (*p == p[-1])
-		    p++;
+                if (*p == '^') {
+                    p++;
+                    type = TOKEN_DBL_XOR;
+                }
 		break;
 
 	    default:
@@ -1647,16 +1647,15 @@ static Token *tokenize(const char *line)
 	    }
         }
 
-        if (type == TOK_WHITESPACE) {
-            *tail = t = new_White(NULL);
-            tail = &t->next;
-        } else if (type != TOK_COMMENT) {
-            if (!ep)
-                ep = p;
-            *tail = t = new_Token(NULL, type, line, ep - line);
-            *tok_text_buf(t) = firstchar; /* E.g. %{foo} -> {foo -> %foo */
-            tail = &t->next;
-        }
+        if (type == TOKEN_EOS)
+            break;              /* done with the string... */
+
+        if (!ep)
+            ep = p;
+        *tail = t = new_Token(NULL, type, line, ep - line);
+        *tok_text_buf(t) = firstchar; /* E.g. %{foo} -> {foo -> %foo */
+        tail = &t->next;
+
         line = p;
     }
     return list;
@@ -1690,7 +1689,7 @@ static Token *alloc_Token(void)
          * block allocations and is not used for data.
          */
         block[0].next = tokenblocks;
-	block[0].type = TOK_BLOCK;
+	block[0].type = TOKEN_BLOCK;
         tokenblocks = block;
 
         /*
@@ -1717,6 +1716,7 @@ static Token *delete_Token(Token *t)
     Token *next = t->next;
 
     nasm_zero(*t);
+    t->type = TOKEN_FREE;
     t->next = freeTokens;
     freeTokens = t;
 
@@ -1760,7 +1760,7 @@ static inline void delete_Blocks(void)
  *  this function creates a new Token and passes a pointer to it
  *  back to the caller.  It sets the type, text, and next pointer elements.
  */
-static Token *new_Token(Token * next, enum pp_token_type type,
+static Token *new_Token(Token * next, enum token_type type,
                         const char *text, size_t txtlen)
 {
     Token *t = alloc_Token();
@@ -1768,7 +1768,7 @@ static Token *new_Token(Token * next, enum pp_token_type type,
 
     t->next = next;
     t->type = type;
-    if (type == TOK_WHITESPACE) {
+    if (type == TOKEN_WHITESPACE) {
         t->len = 1;
         t->text.a[0] = ' ';
     } else {
@@ -1802,7 +1802,7 @@ static Token *new_Token(Token * next, enum pp_token_type type,
  * either taken over or freed.  This function MUST be called
  * with valid txt and txtlen, unlike new_Token().
  */
-static Token *new_Token_free(Token * next, enum pp_token_type type,
+static Token *new_Token_free(Token * next, enum token_type type,
                              char *text, size_t txtlen)
 {
     Token *t = alloc_Token();
@@ -1841,7 +1841,7 @@ static Token *new_White(Token *next)
     Token *t = alloc_Token();
 
     t->next = next;
-    t->type = TOK_WHITESPACE;
+    t->type = TOKEN_WHITESPACE;
     t->len  = 1;
     t->text.a[0] = ' ';
 
@@ -1881,16 +1881,16 @@ static char *detoken(Token * tlist, bool expand_locals)
 
     list_for_each(t, tlist) {
 	switch (t->type) {
-	case TOK_ENVIRON:
+	case TOKEN_ENVIRON:
 	{
 	    const char *v = pp_getenv(t, true);
 	    set_text(t, v, tok_strlen(v));
-	    t->type = TOK_NAKED_STRING;
+	    t->type = TOKEN_NAKED_STR;
 	    break;
         }
 
-	case TOK_LOCAL_MACRO:
-        case TOK_LOCAL_SYMBOL:
+	case TOKEN_LOCAL_MACRO:
+        case TOKEN_LOCAL_SYMBOL:
 	    if (expand_locals) {
 		const char *q;
 		char *p;
@@ -1898,7 +1898,7 @@ static char *detoken(Token * tlist, bool expand_locals)
 		if (ctx) {
 		    p = nasm_asprintf("..@%"PRIu64".%s", ctx->number, q);
 		    set_text_free(t, p, nasm_last_string_len());
-		    t->type = TOK_ID;
+		    t->type = TOKEN_ID;
 		}
 	    }
 	    break;
@@ -1957,31 +1957,20 @@ static int ppscan(void *private_data, struct tokenval *tokval)
 	    pps->ntokens = 0;
 	    return tokval->t_type = TOKEN_EOS;
 	}
-    } while (tline->type == TOK_WHITESPACE || tline->type == TOK_COMMENT);
+    } while (tline->type == TOKEN_WHITESPACE);
 
     txt = tok_text(tline);
     tokval->t_charptr = (char *)txt; /* Fix this */
 
-    if (txt[0] == '$') {
-	if (!txt[1]) {
-	    return tokval->t_type = TOKEN_HERE;
-	} else if (txt[1] == '$' && !txt[2]) {
-	    return tokval->t_type = TOKEN_BASE;
-	} else if (tline->type == TOK_ID) {
-	    tokval->t_charptr++;
-	    return tokval->t_type = TOKEN_ID;
-	}
-    }
-
     switch (tline->type) {
     default:
-	if (tline->len == 1)
-	    return tokval->t_type = txt[0];
-	/* fall through */
-    case TOK_ID:
+        return tokval->t_type = tline->type;
+
+    case TOKEN_ID:
+        /* This could be an assembler keyword */
 	return nasm_token_hash(txt, tokval);
 
-    case TOK_NUMBER:
+    case TOKEN_NUM:
     {
         bool rn_error;
         tokval->t_integer = readnum(txt, &rn_error);
@@ -1991,11 +1980,11 @@ static int ppscan(void *private_data, struct tokenval *tokval)
             return tokval->t_type = TOKEN_NUM;
     }
 
-    case TOK_FLOAT:
-        return tokval->t_type = TOKEN_FLOAT;
-
-    case TOK_STRING:
+    case TOKEN_STR:
 	tokval->t_charptr = (char *)unquote_token(tline);
+        /* fall through */
+    case TOKEN_INTERNAL_STR:
+    case TOKEN_NAKED_STR:
         tokval->t_inttwo = tline->len;
 	return tokval->t_type = TOKEN_STR;
     }
@@ -2024,7 +2013,7 @@ static bool pp_get_boolean_option(Token *tline, bool defval)
     if (!tline)
         return true;
 
-    if (tline->type == TOK_ID) {
+    if (tline->type == TOKEN_ID) {
         size_t i;
 	const char *txt = tok_text(tline);
 
@@ -2441,7 +2430,7 @@ static enum cond_state if_condition(Token * tline, enum preproc_token ct)
     struct ppscan pps;
     struct tokenval tokval;
     expr *evalresult;
-    enum pp_token_type needtype;
+    enum token_type needtype;
     const char *dname = pp_directives[ct];
     bool casesense = true;
     enum preproc_token cond = PP_COND(ct);
@@ -2455,7 +2444,7 @@ static enum cond_state if_condition(Token * tline, enum preproc_token ct)
             tline = skip_white(tline);
             if (!tline)
                 break;
-            if (tline->type != TOK_ID) {
+            if (tline->type != TOKEN_ID) {
                 nasm_nonfatal("`%s' expects context identifiers",
                               dname);
                 goto fail;
@@ -2477,8 +2466,8 @@ static enum cond_state if_condition(Token * tline, enum preproc_token ct)
         j = false;              /* have we matched yet? */
         while (tline) {
             tline = skip_white(tline);
-            if (!tline || (tline->type != TOK_ID &&
-			   tline->type != TOK_LOCAL_MACRO)) {
+            if (!tline || (tline->type != TOKEN_ID &&
+			   tline->type != TOKEN_LOCAL_MACRO)) {
                 nasm_nonfatal("`%s' expects macro identifiers",
                               dname);
                 goto fail;
@@ -2510,10 +2499,10 @@ static enum cond_state if_condition(Token * tline, enum preproc_token ct)
         j = false;              /* have we matched yet? */
         while (tline) {
             tline = skip_white(tline);
-            if (!tline || (tline->type != TOK_ID &&
-                           tline->type != TOK_STRING &&
-			   tline->type != TOK_INTERNAL_STRING &&
-                           tline->type != TOK_ENVIRON)) {
+            if (!tline || (tline->type != TOKEN_ID &&
+                           tline->type != TOKEN_STR &&
+			   tline->type != TOKEN_INTERNAL_STR &&
+                           tline->type != TOKEN_ENVIRON)) {
                 nasm_nonfatal("`%s' expects environment variable names",
                               dname);
                 goto fail;
@@ -2548,11 +2537,11 @@ static enum cond_state if_condition(Token * tline, enum preproc_token ct)
                               dname);
                 goto fail;
             }
-            if (t->type == TOK_WHITESPACE) {
+            if (t->type == TOKEN_WHITESPACE) {
                 t = t->next;
                 continue;
             }
-            if (tt->type == TOK_WHITESPACE) {
+            if (tt->type == TOKEN_WHITESPACE) {
                 tt = tt->next;
                 continue;
             }
@@ -2585,7 +2574,7 @@ static enum cond_state if_condition(Token * tline, enum preproc_token ct)
 
         tline = skip_white(tline);
         tline = expand_id(tline);
-        if (!tok_type(tline, TOK_ID)) {
+        if (!tok_is(tline, TOKEN_ID)) {
             nasm_nonfatal("`%s' expects a macro name", dname);
             goto fail;
         }
@@ -2597,7 +2586,7 @@ static enum cond_state if_condition(Token * tline, enum preproc_token ct)
         tline = expand_smacro(tline->next);
         tline = skip_white(tline);
         if (!tline) {
-        } else if (!tok_type(tline, TOK_NUMBER)) {
+        } else if (!tok_is(tline, TOKEN_NUM)) {
             nasm_nonfatal("`%s' expects a parameter count or nothing",
                           dname);
         } else {
@@ -2608,7 +2597,7 @@ static enum cond_state if_condition(Token * tline, enum preproc_token ct)
             tline = tline->next->next;
             if (tok_is(tline, '*'))
                 searching.nparam_max = INT_MAX;
-            else if (!tok_type(tline, TOK_NUMBER))
+            else if (!tok_is(tline, TOKEN_NUM))
                 nasm_nonfatal("`%s' expects a parameter count after `-'",
                               dname);
             else {
@@ -2643,23 +2632,23 @@ static enum cond_state if_condition(Token * tline, enum preproc_token ct)
     }
 
     case PP_IFID:
-        needtype = TOK_ID;
+        needtype = TOKEN_ID;
         goto iftype;
     case PP_IFNUM:
-        needtype = TOK_NUMBER;
+        needtype = TOKEN_NUM;
         goto iftype;
     case PP_IFSTR:
-        needtype = TOK_STRING;
+        needtype = TOKEN_STR;
         goto iftype;
 
 iftype:
         t = tline = expand_smacro(tline);
 
         while (tok_white(t) ||
-               (needtype == TOK_NUMBER && (tok_is(t, '-') | tok_is(t, '+'))))
+               (needtype == TOKEN_NUM && (tok_is(t, '-') | tok_is(t, '+'))))
             t = t->next;
 
-        j = tok_type(t, needtype);
+        j = tok_is(t, needtype);
         break;
 
     case PP_IFTOKEN:
@@ -2881,7 +2870,7 @@ static int parse_smacro_template(Token ***tpp, SMacro *tmpl)
     err = done = false;
 
     while (!done) {
-        if (!t || !t->type) {
+        if (!t) {
             if (name || flags)
                 nasm_nonfatal("`)' expected to terminate macro template");
             else
@@ -2890,56 +2879,46 @@ static int parse_smacro_template(Token ***tpp, SMacro *tmpl)
         }
 
         switch (t->type) {
-        case TOK_ID:
+        case TOKEN_ID:
             if (name)
                 goto bad;
             name = t;
             break;
-
-        case TOK_OTHER:
-            if (t->len != 1)
-                goto bad;
-            switch (t->text.a[0]) {
-            case '=':
-                flags |= SPARM_EVAL;
-                break;
-            case '&':
-                flags |= SPARM_STR;
-                break;
-            case '!':
-                flags |= SPARM_NOSTRIP;
-                break;
-            case '+':
-                flags |= SPARM_GREEDY;
-                greedy = true;
-                break;
-            case ',':
-                if (greedy)
-                    nasm_nonfatal("greedy parameter must be last");
-                /* fall through */
-            case ')':
-                if (params) {
-                    if (name)
-			steal_Token(&params[nparam].name, name);
-                    params[nparam].flags = flags;
-                }
-                nparam++;
-                name = NULL;
-                flags = 0;
-                done = t->text.a[0] == ')';
-                break;
-            default:
-                goto bad;
+        case '=':
+            flags |= SPARM_EVAL;
+            break;
+        case '&':
+            flags |= SPARM_STR;
+            break;
+        case '!':
+            flags |= SPARM_NOSTRIP;
+            break;
+        case '+':
+            flags |= SPARM_GREEDY;
+            greedy = true;
+            break;
+        case ',':
+            if (greedy)
+                nasm_nonfatal("greedy parameter must be last");
+            /* fall through */
+        case ')':
+            if (params) {
+                if (name)
+                    steal_Token(&params[nparam].name, name);
+                params[nparam].flags = flags;
             }
+            nparam++;
+            name = NULL;
+            flags = 0;
+            done = true;
             break;
-
-        case TOK_WHITESPACE:
+        case TOKEN_WHITESPACE:
             break;
-
         default:
         bad:
             if (!err) {
-                nasm_nonfatal("garbage `%s' in macro parameter list", tok_text(t));
+                nasm_nonfatal("garbage `%s' in macro parameter list",
+                              tok_text(t));
                 err = true;
             }
             break;
@@ -2950,7 +2929,7 @@ static int parse_smacro_template(Token ***tpp, SMacro *tmpl)
     }
 
 finish:
-    while (t && t->type == TOK_WHITESPACE) {
+    while (t && t->type == TOKEN_WHITESPACE) {
         tn = &t->next;
         t = t->next;
     }
@@ -3105,7 +3084,7 @@ static bool parse_mmacro_spec(Token *tline, MMacro *def, const char *directive)
     tline = tline->next;
     tline = skip_white(tline);
     tline = expand_id(tline);
-    if (!tok_type(tline, TOK_ID)) {
+    if (!tok_is(tline, TOKEN_ID)) {
         nasm_nonfatal("`%s' expects a macro name", directive);
         return false;
     }
@@ -3121,7 +3100,7 @@ static bool parse_mmacro_spec(Token *tline, MMacro *def, const char *directive)
 
     tline = expand_smacro(tline->next);
     tline = skip_white(tline);
-    if (!tok_type(tline, TOK_NUMBER))
+    if (!tok_is(tline, TOKEN_NUM))
         nasm_nonfatal("`%s' expects a parameter count", directive);
     else
         def->nparam_min = def->nparam_max = read_param_count(tok_text(tline));
@@ -3129,7 +3108,7 @@ static bool parse_mmacro_spec(Token *tline, MMacro *def, const char *directive)
         tline = tline->next->next;
         if (tok_is(tline, '*')) {
             def->nparam_max = INT_MAX;
-        } else if (!tok_type(tline, TOK_NUMBER)) {
+        } else if (!tok_is(tline, TOKEN_NUM)) {
             nasm_nonfatal("`%s' expects a parameter count after `-'", directive);
         } else {
             def->nparam_max = read_param_count(tok_text(tline));
@@ -3143,7 +3122,7 @@ static bool parse_mmacro_spec(Token *tline, MMacro *def, const char *directive)
         tline = tline->next;
         def->plus = true;
     }
-    if (tline && tok_type(tline->next, TOK_ID) &&
+    if (tline && tok_is(tline->next, TOKEN_ID) &&
 	tline->next->len == 7 &&
         !nasm_stricmp(tline->next->text.a, ".nolist")) {
         tline = tline->next;
@@ -3212,7 +3191,7 @@ static void do_pragma_preproc(Token *tline)
     tline = tline->next;
     tline = skip_white(tline);
 
-    if (!tok_type(tline, TOK_ID))
+    if (!tok_is(tline, TOKEN_ID))
         return;
 
     txt = tok_text(tline);
@@ -3227,7 +3206,7 @@ static void do_pragma_preproc(Token *tline)
 
 static bool is_macro_id(const Token *t)
 {
-    return tok_type(t, TOK_ID) || tok_type(t, TOK_LOCAL_MACRO);
+    return tok_is(t, TOKEN_ID) || tok_is(t, TOKEN_LOCAL_MACRO);
 }
 
 static const char *get_id(Token **tp, const char *dname)
@@ -3264,7 +3243,7 @@ get_use_pkg(Token *t, const char *dname, const char **name)
     if (!t) {
         nasm_nonfatal("`%s' expects a package name, got end of line", dname);
         return NULL;
-    } else if (t->type != TOK_ID && t->type != TOK_STRING) {
+    } else if (t->type != TOKEN_ID && t->type != TOKEN_STR) {
         nasm_nonfatal("`%s' expects a package name, got `%s'",
                       dname, tok_text(t));
         return NULL;
@@ -3283,13 +3262,13 @@ get_use_pkg(Token *t, const char *dname, const char **name)
 /*
  * Mark parameter tokens in an smacro definition. If the type argument
  * is 0, create smac param tokens, otherwise use the type specified;
- * normally this is used for TOK_XDEF_PARAM, which is used to protect
+ * normally this is used for TOKEN_XDEF_PARAM, which is used to protect
  * parameter tokens during expansion during %xdefine.
  *
  * tmpl may not be NULL here.
  */
 static void mark_smac_params(Token *tline, const SMacro *tmpl,
-                             enum pp_token_type type)
+                             enum token_type type)
 {
     const struct smac_param *params = tmpl->params;
     int nparam = tmpl->nparam;
@@ -3297,7 +3276,7 @@ static void mark_smac_params(Token *tline, const SMacro *tmpl,
     int i;
 
     list_for_each(t, tline) {
-        if (t->type != TOK_ID && t->type != TOK_XDEF_PARAM)
+        if (t->type != TOKEN_ID && t->type != TOKEN_XDEF_PARAM)
             continue;
 
         for (i = 0; i < nparam; i++) {
@@ -3353,7 +3332,7 @@ static int line_directive(Token *origline, Token *tline)
     dname = tok_text(tline);
     tline = tline->next;
     tline = skip_white(tline);
-    if (!tok_type(tline, TOK_NUMBER)) {
+    if (!tok_is(tline, TOKEN_NUM)) {
         nasm_nonfatal("`%s' expects a line number", dname);
         goto done;
     }
@@ -3363,7 +3342,7 @@ static int line_directive(Token *origline, Token *tline)
     if (tok_is(tline, '+') || tok_is(tline, '-')) {
         bool minus = tok_is(tline, '-');
         tline = tline->next;
-        if (!tok_type(tline, TOK_NUMBER)) {
+        if (!tok_is(tline, TOKEN_NUM)) {
             nasm_nonfatal("`%s' expects a line increment", dname);
             goto done;
         }
@@ -3374,7 +3353,7 @@ static int line_directive(Token *origline, Token *tline)
     }
     tline = skip_white(tline);
     if (tline) {
-        if (tline->type == TOK_STRING) {
+        if (tline->type == TOKEN_STR) {
             const char *fname;
             /*
              * If this is a quoted string, ignore anything after
@@ -3414,7 +3393,7 @@ static void define_stack_smacro(const char *name, int offset)
     tt = make_tok_num(tt, offset);
     if (!tok_is(tt, '-'))
         tt = make_tok_char(tt, '+');
-    tt = new_Token(tt, TOK_ID, StackPointer, 0);
+    tt = new_Token(tt, TOKEN_ID, StackPointer, 0);
     tt = make_tok_char(tt, '(');
 
     define_smacro(name, true, tt, NULL);
@@ -3509,7 +3488,7 @@ static int do_directive(Token *tline, Token **output)
         return NO_DIRECTIVE_FOUND;
 
     switch (tline->type) {
-    case TOK_PREPROC_ID:
+    case TOKEN_PREPROC_ID:
         dname = tok_text(tline);
         /*
          * For it to be a directive, the second character has to be an
@@ -3522,7 +3501,7 @@ static int do_directive(Token *tline, Token **output)
         op = pp_token_hash(dname);
         break;
 
-    case TOK_ID:
+    case TOKEN_ID:
         if (likely(!(ppopt & PP_TASM)))
             return NO_DIRECTIVE_FOUND;
 
@@ -3626,7 +3605,7 @@ static int do_directive(Token *tline, Token **output)
         tline = tline->next;
         t->next = NULL;
         tline = zap_white(expand_smacro(tline));
-        if (tok_type(tline, TOK_ID)) {
+        if (tok_is(tline, TOKEN_ID)) {
             if (!nasm_stricmp(tok_text(tline), "preproc")) {
                 /* Preprocessor pragma */
                 do_pragma_preproc(tline);
@@ -3641,7 +3620,7 @@ static int do_directive(Token *tline, Token **output)
 
                 /* Prepend "[pragma " */
                 t = new_White(tline);
-                t = new_Token(t, TOK_ID, "pragma", 6);
+                t = new_Token(t, TOKEN_ID, "pragma", 6);
                 t = make_tok_char(t, '[');
                 tline = t;
                 *output = tline;
@@ -3655,7 +3634,7 @@ static int do_directive(Token *tline, Token **output)
          * %stacksize large.
          */
         tline = skip_white(tline->next);
-        if (!tline || tline->type != TOK_ID) {
+        if (!tline || tline->type != TOKEN_ID) {
             nasm_nonfatal("`%s' missing size parameter", dname);
         }
         if (nasm_stricmp(tok_text(tline), "flat") == 0) {
@@ -3704,7 +3683,7 @@ static int do_directive(Token *tline, Token **output)
 
             /* Find the argument name */
             tline = skip_white(tline->next);
-            if (!tline || tline->type != TOK_ID) {
+            if (!tline || tline->type != TOKEN_ID) {
                 nasm_nonfatal("`%s' missing argument parameter", dname);
                 goto done;
             }
@@ -3717,7 +3696,7 @@ static int do_directive(Token *tline, Token **output)
                 goto done;
             }
             tline = tline->next;
-            if (!tok_type(tline, TOK_ID)) {
+            if (!tok_is(tline, TOKEN_ID)) {
                 nasm_nonfatal("`%s' missing size type parameter", dname);
                 goto done;
             }
@@ -3766,7 +3745,7 @@ static int do_directive(Token *tline, Token **output)
 
             /* Find the argument name */
             tline = skip_white(tline->next);
-            if (!tline || tline->type != TOK_ID) {
+            if (!tline || tline->type != TOKEN_ID) {
                 nasm_nonfatal("`%s' missing argument parameter", dname);
                 goto done;
             }
@@ -3779,7 +3758,7 @@ static int do_directive(Token *tline, Token **output)
                 goto done;
             }
             tline = tline->next;
-            if (!tok_type(tline, TOK_ID)) {
+            if (!tok_is(tline, TOKEN_ID)) {
                 nasm_nonfatal("`%s' missing size type parameter", dname);
                 goto done;
             }
@@ -3813,7 +3792,7 @@ static int do_directive(Token *tline, Token **output)
         /* Now define the assign to setup the enter_c macro correctly */
         tt = make_tok_num(NULL, total_size);
         tt = make_tok_char(tt, '+');
-        tt = new_Token(tt, TOK_LOCAL_MACRO, "%$localsize", 11);
+        tt = new_Token(tt, TOKEN_LOCAL_MACRO, "%$localsize", 11);
         assign_smacro("%$localsize", true, tt, dname);
 
         LocalOffset = offset;
@@ -3829,7 +3808,7 @@ static int do_directive(Token *tline, Token **output)
             /* Emulate legacy behavior */
             do_clear(CLEAR_DEFINE|CLEAR_MMACRO, false);
         } else {
-            while ((t = skip_white(t)) && t->type == TOK_ID) {
+            while ((t = skip_white(t)) && t->type == TOKEN_ID) {
                 const char *txt = tok_text(t);
                 if (!nasm_stricmp(txt, "all")) {
                     do_clear(CLEAR_ALL, context);
@@ -3874,8 +3853,8 @@ static int do_directive(Token *tline, Token **output)
     case PP_DEPEND:
         t = tline->next = expand_smacro(tline->next);
         t = skip_white(t);
-        if (!t || (t->type != TOK_STRING &&
-                   t->type != TOK_INTERNAL_STRING)) {
+        if (!t || (t->type != TOKEN_STR &&
+                   t->type != TOKEN_INTERNAL_STR)) {
             nasm_nonfatal("`%s' expects a file name", dname);
             goto done;
         }
@@ -3890,8 +3869,8 @@ static int do_directive(Token *tline, Token **output)
         t = tline->next = expand_smacro(tline->next);
         t = skip_white(t);
 
-        if (!t || (t->type != TOK_STRING &&
-                   t->type != TOK_INTERNAL_STRING)) {
+        if (!t || (t->type != TOKEN_STR &&
+                   t->type != TOKEN_INTERNAL_STR)) {
             nasm_nonfatal("`%s' expects a file name", dname);
             goto done;
         }
@@ -3960,7 +3939,7 @@ static int do_directive(Token *tline, Token **output)
         tline = skip_white(tline);
         tline = expand_id(tline);
         if (tline) {
-            if (!tok_type(tline, TOK_ID)) {
+            if (!tok_is(tline, TOKEN_ID)) {
                 nasm_nonfatal("`%s' expects a context identifier", dname);
                 goto done;
             }
@@ -4020,7 +3999,7 @@ issue_error:
         tline = skip_white(tline);
         t = tline ? tline->next : NULL;
         t = skip_white(t);
-        if (tok_type(tline, TOK_STRING) && !t) {
+        if (tok_is(tline, TOKEN_STR) && !t) {
             /* The line contains only a quoted string */
             p = unquote_token(tline); /* Ignore NUL character truncation */
             nasm_error(severity, "%s",  p);
@@ -4274,7 +4253,7 @@ issue_error:
 
         nolist = 0;
         tline = skip_white(tline->next);
-        if (tok_type(tline, TOK_ID) && tline->len == 7 &&
+        if (tok_is(tline, TOKEN_ID) && tline->len == 7 &&
 	    !nasm_memicmp(tline->text.a, ".nolist", 7)) {
             if (!list_option('f'))
                 nolist |= NL_LIST; /* ... but update line numbers */
@@ -4416,7 +4395,7 @@ issue_error:
             if (op == PP_XDEFINE) {
                 /* Protect macro parameter tokens */
                 if (nparam)
-                    mark_smac_params(tline, &tmpl, TOK_XDEF_PARAM);
+                    mark_smac_params(tline, &tmpl, TOKEN_XDEF_PARAM);
                 tline = expand_smacro(tline);
             }
             macro_start = tline;
@@ -4475,7 +4454,7 @@ issue_error:
 
         t = skip_white(tline);
         /* t should now point to the string */
-        if (!tok_type(t, TOK_STRING)) {
+        if (!tok_is(t, TOKEN_STR)) {
             nasm_nonfatal("`%s' requires string as second parameter", dname);
             free_tlist(tline);
             goto done;
@@ -4509,8 +4488,8 @@ issue_error:
         last->next = NULL;
 
         t = skip_white(tline);
-        if (!t || (t->type != TOK_STRING &&
-                   t->type != TOK_INTERNAL_STRING)) {
+        if (!t || (t->type != TOKEN_STR &&
+                   t->type != TOKEN_INTERNAL_STR)) {
             nasm_nonfatal("`%s' expects a file name", dname);
             free_tlist(tline);
             goto done;
@@ -4545,7 +4524,7 @@ issue_error:
 
         t = skip_white(tline);
         /* t should now point to the string */
-        if (!tok_type(t, TOK_STRING)) {
+        if (!tok_is(t, TOKEN_STR)) {
             nasm_nonfatal("`%s' requires string as second parameter", dname);
             free_tlist(tline);
             free_tlist(origline);
@@ -4576,13 +4555,13 @@ issue_error:
         len = 0;
         list_for_each(t, tline) {
             switch (t->type) {
-            case TOK_WHITESPACE:
+            case TOKEN_WHITESPACE:
                 break;
-            case TOK_STRING:
+            case TOKEN_STR:
 		unquote_token(t);
                 len += t->len;
                 break;
-            case TOK_OTHER:
+            case TOKEN_OTHER:
                 if (tok_is(t, ',')) /* permit comma separators */
                     break;
                 /* else fall through */
@@ -4596,7 +4575,7 @@ issue_error:
 
         q = qbuf = nasm_malloc(len+1);
         list_for_each(t, tline) {
-            if (t->type == TOK_INTERNAL_STRING)
+            if (t->type == TOKEN_INTERNAL_STR)
                 q = mempcpy(q, tok_text(t), t->len);
         }
         *q = '\0';
@@ -4631,7 +4610,7 @@ issue_error:
         t = skip_white(t);
 
         /* t should now point to the string */
-        if (!tok_type(t, TOK_STRING)) {
+        if (!tok_is(t, TOKEN_STR)) {
             nasm_nonfatal("`%s' requires string as second parameter", dname);
             free_tlist(tline);
             goto done;
@@ -4734,7 +4713,7 @@ static int find_cc(Token * t)
         return -1;              /* Probably a %+ without a space */
 
     t = skip_white(t);
-    if (!tok_type(t, TOK_ID))
+    if (!tok_is(t, TOKEN_ID))
         return -1;
     tt = t->next;
     tt = skip_white(tt);
@@ -4745,9 +4724,57 @@ static int find_cc(Token * t)
 		ARRAY_SIZE(conditions));
 }
 
-static inline bool pp_concat_match(const Token *t, unsigned int mask)
+enum concat_flags {
+    CONCAT_ID             = 0x01,
+    CONCAT_LOCAL_MACRO    = 0x02,
+    CONCAT_ENVIRON        = 0x04,
+    CONCAT_PREPROC_ID     = 0x08,
+    CONCAT_NUM            = 0x10,
+    CONCAT_FLOAT          = 0x20,
+    CONCAT_OP             = 0x40  /* Operators */
+};
+
+struct concat_mask {
+    enum concat_flags mask_head;
+    enum concat_flags mask_tail;
+};
+
+
+static inline bool pp_concat_match(const Token *t, enum concat_flags mask)
 {
-    return t && (PP_CONCAT_MASK(t->type) & mask);
+    enum concat_flags ctype = 0;
+
+    if (!t)
+        return false;
+
+    switch (t->type) {
+    case TOKEN_ID:
+        ctype = CONCAT_ID;      /* Should this include $ and $$? */
+        break;
+    case TOKEN_LOCAL_MACRO:
+        ctype = CONCAT_LOCAL_MACRO;
+        break;
+    case TOKEN_ENVIRON:
+        ctype = CONCAT_ENVIRON;
+        break;
+    case TOKEN_PREPROC_ID:
+        ctype = CONCAT_PREPROC_ID;
+        break;
+    case TOKEN_NUM:
+    case TOKEN_FLOAT:
+        ctype = CONCAT_NUM;
+        break;
+    case TOKEN_OTHER:
+        ctype = CONCAT_OP;      /* For historical reasons */
+        break;
+    default:
+        if (t->type > TOKEN_WHITESPACE && t->type < TOKEN_MAX_OPERATOR)
+            ctype = CONCAT_OP;
+        else
+            ctype = 0;
+    }
+
+    return !!(ctype & mask);
 }
 
 /*
@@ -4757,7 +4784,7 @@ static inline bool pp_concat_match(const Token *t, unsigned int mask)
  * The @m array can contain a series of token types which are
  * executed as separate passes.
  */
-static bool paste_tokens(Token **head, const struct tokseq_match *m,
+static bool paste_tokens(Token **head, const struct concat_mask *m,
                          size_t mnum, bool handle_explicit)
 {
     Token *tok, *t, *next, **prev_next, **prev_nonspace;
@@ -4780,19 +4807,19 @@ static bool paste_tokens(Token **head, const struct tokseq_match *m,
     tok = *head;
     prev_next = prev_nonspace = head;
 
-    if (tok_white(tok) || tok_type(tok, TOK_PASTE))
+    if (tok_white(tok) || tok_is(tok, TOKEN_PASTE))
         prev_nonspace = NULL;
 
     while (tok && (next = tok->next)) {
         bool did_paste = false;
 
         switch (tok->type) {
-        case TOK_WHITESPACE:
+        case TOKEN_WHITESPACE:
             /* Zap redundant whitespaces */
             tok->next = next = zap_white(next);
             break;
 
-        case TOK_PASTE:
+        case TOKEN_PASTE:
             /* Explicit pasting */
             if (!handle_explicit)
                 break;
@@ -4819,7 +4846,7 @@ static bool paste_tokens(Token **head, const struct tokseq_match *m,
              * just drop whem in that case.
              */
             while (next) {
-                if (next->type == TOK_PASTE || next->type == TOK_WHITESPACE)
+                if (next->type == TOKEN_PASTE || next->type == TOKEN_WHITESPACE)
                     next = delete_Token(next);
                 else
                     break;
@@ -4905,7 +4932,7 @@ static bool paste_tokens(Token **head, const struct tokseq_match *m,
             pasted = true;
         } else {
             prev_next = &tok->next;
-            if (next && next->type != TOK_WHITESPACE && next->type != TOK_PASTE)
+            if (next && next->type != TOKEN_WHITESPACE && next->type != TOKEN_PASTE)
                 prev_nonspace = prev_next;
         }
 
@@ -5004,7 +5031,7 @@ static Token *expand_mmac_params(Token * tline)
         t->next = NULL;
 
         switch (type) {
-        case TOK_LOCAL_SYMBOL:
+        case TOKEN_LOCAL_SYMBOL:
             change = true;
 
             if (!mac) {
@@ -5012,10 +5039,10 @@ static Token *expand_mmac_params(Token * tline)
                 break;
             }
 
-            type = TOK_ID;
+            type = TOKEN_ID;
             text = nasm_asprintf("..@%"PRIu64".%s", mac->unique, text+2);
             break;
-        case TOK_MMACRO_PARAM:
+        case TOKEN_MMACRO_PARAM:
         {
             Token *tt = NULL;
 
@@ -5040,7 +5067,7 @@ static Token *expand_mmac_params(Token * tline)
                  */
             case '0':
                 if (!text[2]) {
-                    type = TOK_NUMBER;
+                    type = TOKEN_NUM;
                     text = nasm_asprintf("%d", mac->nparam);
                     break;
                 }
@@ -5087,7 +5114,7 @@ static Token *expand_mmac_params(Token * tline)
                     break;
                 }
 
-                type = TOK_ID;
+                type = TOKEN_ID;
                 if (text[1] == '-') {
                     int ncc = inverse_ccs[cc];
                     if (unlikely(ncc == -1)) {
@@ -5109,9 +5136,9 @@ static Token *expand_mmac_params(Token * tline)
             break;
         }
 
-        case TOK_PREPROC_Q:
+        case TOKEN_PREPROC_Q:
             if (mac) {
-                type = TOK_ID;
+                type = TOKEN_ID;
                 text = nasm_strdup(mac->iname);
                 change = true;
             } else {
@@ -5119,9 +5146,9 @@ static Token *expand_mmac_params(Token * tline)
             }
             break;
 
-        case TOK_PREPROC_QQ:
+        case TOKEN_PREPROC_QQ:
             if (mac) {
-                type = TOK_ID;
+                type = TOKEN_ID;
                 text = nasm_strdup(mac->name);
                 change = true;
             } else {
@@ -5129,7 +5156,7 @@ static Token *expand_mmac_params(Token * tline)
             }
             break;
 
-        case TOK_INDIRECT:
+        case TOKEN_INDIRECT:
         {
             Token *tt;
 
@@ -5173,18 +5200,14 @@ static Token *expand_mmac_params(Token * tline)
     *tail = NULL;
 
     if (changed) {
-        const struct tokseq_match t[] = {
+        const struct concat_mask t[] = {
             {
-                PP_CONCAT_MASK(TOK_ID)          |
-                PP_CONCAT_MASK(TOK_FLOAT),          /* head */
-                PP_CONCAT_MASK(TOK_ID)          |
-                PP_CONCAT_MASK(TOK_NUMBER)      |
-                PP_CONCAT_MASK(TOK_FLOAT)       |
-                PP_CONCAT_MASK(TOK_OTHER)           /* tail */
+                CONCAT_ID | CONCAT_FLOAT,   /* head */
+                CONCAT_ID | CONCAT_NUM | CONCAT_FLOAT | CONCAT_OP /* tail */
             },
             {
-                PP_CONCAT_MASK(TOK_NUMBER),         /* head */
-                PP_CONCAT_MASK(TOK_NUMBER)          /* tail */
+                CONCAT_NUM,     /* head */
+                CONCAT_NUM      /* tail */
             }
         };
         paste_tokens(&thead, t, ARRAY_SIZE(t), false);
@@ -5234,9 +5257,9 @@ static SMacro *expand_one_smacro(Token ***tpp)
             smacro_deadman.triggered = true;
         }
         goto not_a_macro;
-    } else if (tline->type == TOK_ID || tline->type == TOK_PREPROC_ID) {
+    } else if (tline->type == TOKEN_ID || tline->type == TOKEN_PREPROC_ID) {
         head = (SMacro *)hash_findix(&smacros, mname);
-    } else if (tline->type == TOK_LOCAL_MACRO) {
+    } else if (tline->type == TOKEN_LOCAL_MACRO) {
         Context *ctx = get_ctx(mname, &mname);
         head = ctx ? (SMacro *)hash_findix(&ctx->localmac, mname) : NULL;
     } else {
@@ -5305,10 +5328,7 @@ static SMacro *expand_one_smacro(Token ***tpp)
                 goto not_a_macro;
             }
 
-            if (t->type != TOK_OTHER || t->len != 1)
-                continue;
-
-            switch (t->text.a[0]) {
+            switch (t->type) {
             case ',':
                 if (!brackets && paren == 1)
                     nparam++;
@@ -5394,24 +5414,16 @@ static SMacro *expand_one_smacro(Token ***tpp)
 
         while (paren) {
             bool skip;
-            char ch;
 
             tline = tline->next;
 
             if (!tline)
                 nasm_nonfatal("macro call expects terminating `)'");
 
-            ch = 0;
             skip = false;
 
-
             switch (tline->type) {
-            case TOK_OTHER:
-                if (tline->len == 1)
-                    ch = tline->text.a[0];
-                break;
-
-            case TOK_WHITESPACE:
+            case TOKEN_WHITESPACE:
                 if (!(flags & SPARM_NOSTRIP)) {
                     if (brackets || *phead)
                         white++;    /* Keep interior whitespace */
@@ -5419,11 +5431,6 @@ static SMacro *expand_one_smacro(Token ***tpp)
                 }
                 break;
 
-            default:
-                break;
-            }
-
-            switch (ch) {
             case ',':
                 if (!brackets && paren == 1 && !(flags & SPARM_GREEDY)) {
                     i++;
@@ -5543,23 +5550,23 @@ static SMacro *expand_one_smacro(Token ***tpp)
     cond_comma = false;
 
     while (t) {
-        enum pp_token_type type = t->type;
+        enum token_type type = t->type;
         Token *tnext = t->next;
 
         switch (type) {
-        case TOK_PREPROC_Q:
+        case TOKEN_PREPROC_Q:
             delete_Token(t);
             t = dup_Token(tline, mstart);
             break;
 
-        case TOK_PREPROC_QQ:
+        case TOKEN_PREPROC_QQ:
         {
             size_t mlen = strlen(m->name);
 	    size_t len;
             char *p;
 
             t->type = mstart->type;
-            if (t->type == TOK_LOCAL_MACRO) {
+            if (t->type == TOKEN_LOCAL_MACRO) {
 		const char *psp; /* prefix start pointer */
                 const char *pep; /* prefix end pointer */
 		size_t plen;
@@ -5583,14 +5590,14 @@ static SMacro *expand_one_smacro(Token ***tpp)
             break;
         }
 
-        case TOK_COND_COMMA:
+        case TOKEN_COND_COMMA:
             delete_Token(t);
             t = cond_comma ? make_tok_char(tline, ',') : NULL;
             break;
 
-        case TOK_ID:
-        case TOK_PREPROC_ID:
-	case TOK_LOCAL_MACRO:
+        case TOKEN_ID:
+        case TOKEN_PREPROC_ID:
+	case TOKEN_LOCAL_MACRO:
         {
             /*
              * Chain this into the target line *before* expanding,
@@ -5622,7 +5629,7 @@ static SMacro *expand_one_smacro(Token ***tpp)
 
             tline = t;
             while (!cond_comma && t && t != endt) {
-                cond_comma = t->type != TOK_WHITESPACE;
+                cond_comma = t->type != TOKEN_WHITESPACE;
                 t = t->next;
             }
         }
@@ -5701,17 +5708,13 @@ static Token *expand_smacro_noreset(Token *org_tline)
     expanded = true;
 
     while (true) {
-        static const struct tokseq_match tmatch[] = {
+        static const struct concat_mask tmatch[] = {
             {
-                PP_CONCAT_MASK(TOK_ID)          |
-		PP_CONCAT_MASK(TOK_LOCAL_MACRO) |
-		PP_CONCAT_MASK(TOK_ENVIRON)     |
-                PP_CONCAT_MASK(TOK_PREPROC_ID),     /* head */
-                PP_CONCAT_MASK(TOK_ID)          |
-		PP_CONCAT_MASK(TOK_LOCAL_MACRO) |
-		PP_CONCAT_MASK(TOK_ENVIRON)     |
-                PP_CONCAT_MASK(TOK_PREPROC_ID)  |
-                PP_CONCAT_MASK(TOK_NUMBER)          /* tail */
+                CONCAT_ID | CONCAT_LOCAL_MACRO |
+                CONCAT_ENVIRON | CONCAT_PREPROC_ID, /* head */
+                CONCAT_ID | CONCAT_LOCAL_MACRO |
+                CONCAT_ENVIRON | CONCAT_PREPROC_ID |
+                CONCAT_NUM      /* tail */
             }
         };
         Token **tail = &tline;
@@ -5730,9 +5733,9 @@ static Token *expand_smacro_noreset(Token *org_tline)
             break;              /* Done! */
 
         /*
-         * Now scan the entire line and look for successive TOK_IDs
+         * Now scan the entire line and look for successive TOKEN_IDs
          * that resulted after expansion (they can't be produced by
-         * tokenize()). The successive TOK_IDs should be concatenated.
+         * tokenize()). The successive TOKEN_IDs should be concatenated.
          * Also we look for %+ tokens and concatenate the tokens
          * before and after them (without white spaces in between).
          */
@@ -5789,8 +5792,10 @@ static Token *expand_id(Token * tline)
 
     cur = tline;
     while (cur->next &&
-           (cur->next->type == TOK_ID || cur->next->type == TOK_PREPROC_ID ||
-	   cur->next->type == TOK_LOCAL_MACRO || cur->next->type == TOK_NUMBER))
+           (cur->next->type == TOKEN_ID ||
+            cur->next->type == TOKEN_PREPROC_ID ||
+            cur->next->type == TOKEN_LOCAL_MACRO ||
+            cur->next->type == TOKEN_NUM))
         cur = cur->next;
 
     /* If identifier consists of just one token, don't expand */
@@ -5907,7 +5912,7 @@ find_mmacro_in_list(MMacro *m, const char *finding,
  * call, and return the MMacro structure called if so. Doesn't have
  * to check for an initial label - that's taken care of in
  * expand_mmacro - but must check numbers of parameters. Guaranteed
- * to be called with tline->type == TOK_ID, so the putative macro
+ * to be called with tline->type == TOKEN_ID, so the putative macro
  * name is easy to find.
  */
 static MMacro *is_mmacro(Token * tline, int *nparamp, Token ***paramsp)
@@ -6156,7 +6161,7 @@ static void list_mmacro_call(const MMacro *m)
         list_for_each(t, m->params[i]) {
             if (j++ >= m->paramlen[i])
                 break;
-            size += (t->type == TOK_WHITESPACE) ? 1 : t->len;
+            size += (t->type == TOKEN_WHITESPACE) ? 1 : t->len;
         }
     }
 
@@ -6351,8 +6356,7 @@ static int expand_mmacro(Token * tline)
 
     t = tline;
     t = skip_white(t);
-    /*    if (!tok_type(t, TOK_ID))  Lino 02/25/02 */
-    if (!tok_type(t, TOK_ID) && !tok_type(t, TOK_LOCAL_MACRO))
+    if (!tok_is(t, TOKEN_ID) && !tok_is(t, TOKEN_LOCAL_MACRO))
         return 0;
     m = is_mmacro(t, &nparam, &params);
     if (m) {
@@ -6375,7 +6379,7 @@ static int expand_mmacro(Token * tline)
             if (tok_white(t))
                 last = t, t = t->next;
         }
-        if (!tok_type(t, TOK_ID) || !(m = is_mmacro(t, &nparam, &params)))
+        if (!tok_is(t, TOKEN_ID) || !(m = is_mmacro(t, &nparam, &params)))
             return 0;
         last->next = NULL;
         mname = tok_text(t);
@@ -6421,28 +6425,26 @@ static int expand_mmacro(Token * tline)
                 continue;
             }
 
-            if (t->type == TOK_OTHER && t->len == 1) {
-                switch (t->text.a[0]) {
-                case ',':
-                    if (comma && !brace)
-                        goto endparam;
-                    break;
-
-                case '{':
-                    brace++;
-                    break;
+            switch(t->type) {
+            case ',':
+                if (comma && !brace)
+                    goto endparam;
+                break;
 
-                case '}':
-                    brace--;
-                    if (braced && !brace) {
-                        paramlen[i] += white;
-                        goto endparam;
-                    }
-                    break;
+            case '{':
+                brace++;
+                break;
 
-                default:
-                    break;
+            case '}':
+                brace--;
+                if (braced && !brace) {
+                    paramlen[i] += white;
+                    goto endparam;
                 }
+                break;
+
+            default:
+                break;
             }
 
             paramlen[i] += white + 1;
@@ -6616,11 +6618,11 @@ stdmac_ptr(const SMacro *s, Token **params, int nparams)
 
     switch (globalbits) {
     case 16:
-	return new_Token(NULL, TOK_ID, "word", 4);
+	return new_Token(NULL, TOKEN_ID, "word", 4);
     case 32:
-	return new_Token(NULL, TOK_ID, "dword", 5);
+	return new_Token(NULL, TOKEN_ID, "dword", 5);
     case 64:
-	return new_Token(NULL, TOK_ID, "qword", 5);
+	return new_Token(NULL, TOKEN_ID, "qword", 5);
     default:
         panic();
     }
@@ -7033,7 +7035,8 @@ static Token *pp_tokline(void)
             if (mmac) {
                 const Token *t;
                 list_for_each(t, tline) {
-                    if (!memcmp(t->text.a, "%00", 4))
+                    if (t->type == TOKEN_PREPROC_ID &&
+                        !memcmp(t->text.a, "%00", 4))
                         mmac->capture_label = true;
                 }
             }
@@ -7146,9 +7149,9 @@ void pp_pre_include(char *fname)
     Token *inc, *space, *name;
     Line *l;
 
-    name = new_Token(NULL, TOK_INTERNAL_STRING, fname, 0);
+    name = new_Token(NULL, TOKEN_INTERNAL_STR, fname, 0);
     space = new_White(name);
-    inc = new_Token(space, TOK_PREPROC_ID, "%include", 0);
+    inc = new_Token(space, TOKEN_PREPROC_ID, "%include", 0);
 
     l = nasm_malloc(sizeof(Line));
     l->next = predef;
@@ -7165,19 +7168,14 @@ void pp_pre_define(char *definition)
 
     equals = strchr(definition, '=');
     space = new_White(NULL);
-    def = new_Token(space, TOK_PREPROC_ID, "%define", 0);
+    def = new_Token(space, TOKEN_PREPROC_ID, "%define", 0);
     if (equals)
         *equals = ' ';
     space->next = tokenize(definition);
     if (equals)
         *equals = '=';
 
-    /* We can't predefine a TOK_LOCAL_MACRO for obvious reasons... */
-    if (space->next->type != TOK_PREPROC_ID &&
-        space->next->type != TOK_ID)
-        nasm_warn(WARN_OTHER, "pre-defining non ID `%s\'\n", definition);
-
-    l = nasm_malloc(sizeof(Line));
+    nasm_new(l);
     l->next = predef;
     l->first = def;
     l->finishes = NULL;
@@ -7190,10 +7188,10 @@ void pp_pre_undefine(char *definition)
     Line *l;
 
     space = new_White(NULL);
-    def = new_Token(space, TOK_PREPROC_ID, "%undef", 0);
+    def = new_Token(space, TOKEN_PREPROC_ID, "%undef", 0);
     space->next = tokenize(definition);
 
-    l = nasm_malloc(sizeof(Line));
+    nasm_new(l);
     l->next = predef;
     l->first = def;
     l->finishes = NULL;
@@ -7203,19 +7201,16 @@ void pp_pre_undefine(char *definition)
 /* Insert an early preprocessor command that doesn't need special handling */
 void pp_pre_command(const char *what, char *string)
 {
-    char *cmd;
     Token *def, *space;
     Line *l;
 
     def = tokenize(string);
     if (what) {
         space = new_White(def);
-        cmd = nasm_strcat(what[0] == '%' ? "" : "%", what);
-        def = new_Token(space, TOK_PREPROC_ID, cmd, nasm_last_string_len());
-        nasm_free(cmd);
+        def = new_Token(space, TOKEN_PREPROC_ID, what, 0);
     }
 
-    l = nasm_malloc(sizeof(Line));
+    nasm_new(l);
     l->next = predef;
     l->first = def;
     l->finishes = NULL;
@@ -7253,7 +7248,7 @@ static Token *make_tok_num(Token *next, int64_t val)
     uval = minus ? -val : val;
 
     len = snprintf(numbuf, sizeof numbuf, "%"PRIu64, uval);
-    next = new_Token(next, TOK_NUMBER, numbuf, len);
+    next = new_Token(next, TOKEN_NUM, numbuf, len);
 
     if (minus)
         next = make_tok_char(next, '-');
@@ -7265,7 +7260,7 @@ static Token *make_tok_num(Token *next, int64_t val)
 static Token *make_tok_qstr_len(Token *next, const char *str, size_t len)
 {
     char *p = nasm_quote(str, &len);
-    return new_Token_free(next, TOK_STRING, p, len);
+    return new_Token_free(next, TOKEN_STR, p, len);
 }
 static Token *make_tok_qstr(Token *next, const char *str)
 {
@@ -7275,7 +7270,7 @@ static Token *make_tok_qstr(Token *next, const char *str)
 /* Create a single-character operator token */
 static Token *make_tok_char(Token *next, char op)
 {
-    Token *t = new_Token(next, TOK_OTHER, NULL, 1);
+    Token *t = new_Token(next, op, NULL, 1);
     t->text.a[0] = op;
     return t;
 }
diff --git a/include/nasm.h b/include/nasm.h
index 3bcd3cda..115084ae 100644
--- a/include/nasm.h
+++ b/include/nasm.h
@@ -155,26 +155,44 @@ typedef void (*ldfunc)(char *label, int32_t segment, int64_t offset,
  * ASCII character values, and zero for end-of-string.
  */
 enum token_type { /* token types, other than chars */
+
+    /* Token values shared between assembler and preprocessor */
+
+    /* Special codes */
     TOKEN_INVALID = -1, /* a placeholder value */
-    TOKEN_EOS = 0,      /* end of string */
-    TOKEN_QMARK = '?',
-    TOKEN_EQ = '=',
-    TOKEN_GT = '>',
-    TOKEN_LT = '<',     /* aliases */
-    TOKEN_ID = 256,     /* identifier */
-    TOKEN_NUM,          /* numeric constant */
-    TOKEN_ERRNUM,       /* malformed numeric constant */
-    TOKEN_STR,          /* string constant */
-    TOKEN_ERRSTR,       /* unterminated string constant */
-    TOKEN_FLOAT,        /* floating-point constant */
-    TOKEN_REG,          /* register name */
-    TOKEN_INSN,         /* instruction name */
-    TOKEN_HERE,         /* $ */
-    TOKEN_BASE,         /* $$ */
-    TOKEN_SIZE,		/* BYTE, WORD, DWORD, QWORD, etc */
-    TOKEN_SPECIAL,      /* REL, FAR, NEAR, STRICT, NOSPLIT, etc */
-    TOKEN_PREFIX,       /* A32, O16, LOCK, REPNZ, TIMES, etc */
-    TOKEN_SHL,          /* << or <<< */
+    TOKEN_BLOCK   = -2, /* used for storage management */
+    TOKEN_FREE    = -3, /* free token marker, use to catch leaks */
+    TOKEN_EOS     = 0,  /* end of string */
+
+    /*
+     * Single-character operators. Enumerated here to keep strict
+     * compilers happy, and for documentation.
+     */
+    TOKEN_WHITESPACE = ' ',     /* Preprocessor use */
+    TOKEN_BOOL_NOT   = '!',
+    TOKEN_AND        = '&',
+    TOKEN_OR         = '|',
+    TOKEN_XOR        = '^',
+    TOKEN_NOT        = '~',
+    TOKEN_MULT       = '*',
+    TOKEN_DIV        = '/',
+    TOKEN_MOD        = '%',
+    TOKEN_LPAR       = ')',
+    TOKEN_RPAR       = '(',
+    TOKEN_PLUS       = '+',
+    TOKEN_MINUS      = '-',
+    TOKEN_COMMA      = ',',
+    TOKEN_LBRACE     = '{',
+    TOKEN_RBRACE     = '}',
+    TOKEN_LBRACKET   = '[',
+    TOKEN_RBRACKET   = ']',
+    TOKEN_QMARK      = '?',
+    TOKEN_EQ         = '=',     /* = or == */
+    TOKEN_GT         = '>',
+    TOKEN_LT         = '<',
+
+    /* Multi-character operators */
+    TOKEN_SHL = 256,    /* << or <<< */
     TOKEN_SHR,          /* >> */
     TOKEN_SAR,          /* >>> */
     TOKEN_SDIV,         /* // */
@@ -186,6 +204,21 @@ enum token_type { /* token types, other than chars */
     TOKEN_DBL_AND,      /* && */
     TOKEN_DBL_OR,       /* || */
     TOKEN_DBL_XOR,      /* ^^ */
+
+    TOKEN_MAX_OPERATOR,
+
+    TOKEN_NUM,          /* numeric constant */
+    TOKEN_ERRNUM,       /* malformed numeric constant */
+    TOKEN_STR,          /* string constant */
+    TOKEN_ERRSTR,       /* unterminated string constant */
+    TOKEN_ID,           /* identifier */
+    TOKEN_FLOAT,        /* floating-point constant */
+    TOKEN_HERE,         /* $, not '$' because it is not an operator */
+    TOKEN_BASE,         /* $$ */
+
+    /* Token values only used by the assembler */
+    TOKEN_START_ASM,
+
     TOKEN_SEG,          /* SEG */
     TOKEN_WRT,          /* WRT */
     TOKEN_FLOATIZE,     /* __?floatX?__ */
@@ -194,7 +227,36 @@ enum token_type { /* token types, other than chars */
     TOKEN_DECORATOR,    /* decorators such as {...} */
     TOKEN_MASM_PTR,     /* __?masm_ptr?__ for the masm package */
     TOKEN_MASM_FLAT,    /* __?masm_flat?__ for the masm package */
-    TOKEN_OPMASK        /* translated token for opmask registers */
+    TOKEN_OPMASK,       /* translated token for opmask registers */
+    TOKEN_SIZE,		/* BYTE, WORD, DWORD, QWORD, etc */
+    TOKEN_SPECIAL,      /* REL, FAR, NEAR, STRICT, NOSPLIT, etc */
+    TOKEN_PREFIX,       /* A32, O16, LOCK, REPNZ, TIMES, etc */
+    TOKEN_REG,          /* register name */
+    TOKEN_INSN,         /* instruction name */
+
+    TOKEN_END_ASM,
+
+    /* Token values only used by the preprocessor */
+
+    TOKEN_START_PP = TOKEN_END_ASM,
+
+    TOKEN_OTHER,           /* % sequence without (current) meaning */
+    TOKEN_PREPROC_ID,      /* Preprocessor ID, e.g. %symbol */
+    TOKEN_MMACRO_PARAM,    /* MMacro parameter, e.g. %1 */
+    TOKEN_LOCAL_SYMBOL,    /* Local symbol, e.g. %%symbol */
+    TOKEN_LOCAL_MACRO,     /* Context-local macro, e.g. %$symbol */
+    TOKEN_ENVIRON,         /* %! */
+    TOKEN_INTERNAL_STR,    /* Unquoted string that should remain so */
+    TOKEN_NAKED_STR,       /* Unquoted string that can be re-quoted */
+    TOKEN_PREPROC_Q,       /* %? */
+    TOKEN_PREPROC_QQ,      /* %?? */
+    TOKEN_PASTE,           /* %+ */
+    TOKEN_COND_COMMA,      /* %, */
+    TOKEN_INDIRECT,        /* %[...] */
+    TOKEN_XDEF_PARAM,      /* Used during %xdefine processing */
+    /* smacro parameters starting here; an arbitrary number. */
+    TOKEN_SMAC_START_PARAMS,    /* MUST BE LAST IN THE LIST!!! */
+    TOKEN_MAX = INT_MAX		/* Keep compiler from reducing the range */
 };
 
 enum floatize {


More information about the Nasm-commits mailing list