[Nasm-bugs] [Bug 3392774] Large define lists crash recent NASM

[noreply-nasm at dev.nasm.us]

https://bugzilla.nasm.us/show_bug.cgi?id=3392774

--- Comment #5 from C. Masloch <pushbx at ulukai.org> ---
> So my strong guess is that either there is a missing token delete somewhere, or the token allocator fails to makes a deleted token head available for reuse.

Wouldn't know about the possibility of leaks. However, the token deletion
appears to work as expected.

Anyway, I made a crude patch to two different NASM revisions to compare their
use of preprocessor tokens. The newer revision is as described here, based on
the commit 3aebb20f123033dcd767f0abc46b18cbefed8091, and patched like this:

diff --git a/asm/preproc.c b/asm/preproc.c
index 7724b12a..203664b5 100644
--- a/asm/preproc.c
+++ b/asm/preproc.c
@@ -1717,6 +1717,19 @@ static Token *tokenize(const char *line)
     return list;
 }

+static FILE * logfile = NULL;
+static unsigned long logamount = 0;
+static unsigned long logtotal = 0;
+static unsigned long logmodulo = 0;
+
+static void openlogfile(void);
+static void openlogfile(void) {
+       if (logfile) return;
+       logfile = fopen("nasmtoka.log", "wb");
+       if (!logfile) nasm_panic("unable to open log file");
+       return;
+}
+
 /*
  * Tokens are allocated in blocks to improve speed. Set the blocksize
  * to 0 to use regular nasm_malloc(); this is useful for debugging.
@@ -1733,6 +1746,13 @@ static Token *tokenblocks = NULL;
 static Token *alloc_Token(void)
 {
     Token *t = freeTokens;
+       openlogfile();
+       logamount++;
+       logtotal++;
+       if ((logmodulo++ & 8191) == 0) {
+               fprintf(logfile, "[%12lu] (%12lu) allocate\n", logamount,
logtotal);
+               fflush(logfile);
+       }

     if (unlikely(!t)) {
         Token *block;
@@ -1770,6 +1790,12 @@ static Token *alloc_Token(void)
 static Token *delete_Token(Token *t)
 {
     Token *next;
+       openlogfile();
+       logamount--;
+       if ((logmodulo++ & 8191) == 0) {
+               fprintf(logfile, "[%12lu] (%12lu) delete\n", logamount,
logtotal);
+               fflush(logfile);
+       }

     nasm_assert(t && t->type != TOKEN_FREE);


Diff also available at https://pushbx.org/ecm/test/20220826/new.diff

The older revision is based on commit 52266ad42490f48b91a70efb5c2f93ea281eeb60
and patched like this:

diff --git a/asm/preproc.c b/asm/preproc.c
index 95ca56fc..26cf3002 100644
--- a/asm/preproc.c
+++ b/asm/preproc.c
@@ -1192,6 +1192,19 @@ static void delete_Blocks(void)
     memset(&blocks, 0, sizeof(blocks));
 }

+static FILE * logfile = NULL;
+static unsigned long logamount = 0;
+static unsigned long logtotal = 0;
+static unsigned long logmodulo = 0;
+
+static void openlogfile(void);
+static void openlogfile(void) {
+       if (logfile) return;
+       logfile = fopen("nasmtoka.log", "wb");
+       if (!logfile) nasm_panic(0, "unable to open log file");
+       return;
+}
+
 /*
  *  this function creates a new Token and passes a pointer to it
  *  back to the caller.  It sets the type and text elements, and
@@ -1202,6 +1215,13 @@ static Token *new_Token(Token * next, enum pp_token_type
type,
 {
     Token *t;
     int i;
+       openlogfile();
+       logamount++;
+       logtotal++;
+       if ((logmodulo++ & 8191) == 0) {
+               fprintf(logfile, "[%12lu] (%12lu) allocate\n", logamount,
logtotal);
+               fflush(logfile);
+       }

     if (!freeTokens) {
         freeTokens = (Token *) new_Block(TOKEN_BLOCKSIZE * sizeof(Token));
@@ -1229,6 +1249,12 @@ static Token *new_Token(Token * next, enum pp_token_type
type,
 static Token *delete_Token(Token * t)
 {
     Token *next = t->next;
+       openlogfile();
+       logamount--;
+       if ((logmodulo++ & 8191) == 0) {
+               fprintf(logfile, "[%12lu] (%12lu) delete\n", logamount,
logtotal);
+               fflush(logfile);
+       }
     nasm_free(t->text);
     t->next = freeTokens;
     freeTokens = t;

Diff also available at https://pushbx.org/ecm/test/20220826/old.diff

The command run with either executable is as follows:

nasm debug.asm -I../../lmacros/ -I../../symsnip/ -I../../scanptab/ -o tmp.bin
-D_DEBUG -D_PM -D_SYMBOLIC -D_DUALCODE -D_SYMBOLASMDUALCODE

Again, the older revision needs 714356 KiB, the newer one needs 2560204 KiB.

These are the ends of the resulting log files:

ldebug/source$ tail nasmtoka.old
[     9739325] (   378949151) delete
[     9738057] (   378952613) allocate
[     9738415] (   378956888) allocate
[     9738479] (   378961016) delete
[     9738607] (   378965176) delete
[     9738789] (   378969363) allocate
[     9739287] (   378973708) delete
[     9739839] (   378978080) delete
[     9740393] (   378982453) allocate
[     9740661] (   378986683) allocate
ldebug/source$ tail nasmtoka.log
[    71368223] (   436961040) allocate
[    71368229] (   436965139) allocate
[    71368351] (   436969296) delete
[    71368459] (   436973446) delete
[    71369195] (   436977910) delete
[    71369851] (   436982334) allocate
[    71370409] (   436986709) delete
[    71370513] (   436990857) allocate
[    71363959] (   436991676) delete
[    71355767] (   436991676) delete
ldebug/source$

The square bracketed numbers show the currently-allocated amount of tokens, the
round parenthetical numbers show the total-allocated amount of tokens (counting
only allocation, not deletion).

So it does appear that the newer revision leaks more than the older. I'm not
sure whether this explains the dramatic memory use increase however.

-- 
You are receiving this mail because:
You are watching all bug changes.
You are on the CC list for the bug.