seg: pure parser and reentrant scanner

Use the flex %option reentrant and the bison option %pure-parser to
make the generated scanner and parser pure, reentrant, and
thread-safe.

Make the generated scanner use palloc() etc. instead of malloc() etc.
Previously, we only used palloc() for the buffer, but flex would still
use malloc() for its internal structures.  As a result, there could be
some small memory s in case of uncaught errors.  (We do catch
normal syntax errors as soft errors.)  Now, all the memory is under
palloc() control, so there are no more such issues.

Simplify flex scan buffer management: Instead of constructing the
buffer from pieces and then using yy_scan_buffer(), we can just use
yy_scan_string(), which does the same thing internally.

The previous code was necessary because we allocated the buffer with
palloc() and the rest of the state was handled by malloc().  But this
is no longer the case; everything is under palloc() now.

(We could even get rid of the yylex_destroy() call and just let the
memory context cleanup handle everything.  But for now, we preserve
the existing behavior.)

Reviewed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Andreas Karlsson <[email protected]>
Discussion: https://www.postgresql.org/message-id/flat/eb6faeac-2a8a-4b69-9189-c33c520e5b7b@eisentraut.org

diff --git a/contrib/seg/seg.c b/contrib/seg/seg.c
index 7f9fc24eb4b60add583d6237e1dd575b17f58293..fd4216edc5dc8d37e436cfee04d3ba433412121c 100644 (file)
--- a/contrib/seg/seg.c
+++ b/contrib/seg/seg.c
@@ -105,13 +105,14 @@ seg_in(PG_FUNCTION_ARGS)
 {
    char       *str = PG_GETARG_CSTRING(0);
    SEG        *result = palloc(sizeof(SEG));
+   yyscan_t    scanner;
 
-   seg_scanner_init(str);
+   seg_scanner_init(str, &scanner);
 
-   if (seg_yyparse(result, fcinfo->context) != 0)
-       seg_yyerror(result, fcinfo->context, "bogus input");
+   if (seg_yyparse(result, fcinfo->context, scanner) != 0)
+       seg_yyerror(result, fcinfo->context, scanner, "bogus input");
 
-   seg_scanner_finish();
+   seg_scanner_finish(scanner);
 
    PG_RETURN_POINTER(result);
 }

diff --git a/contrib/seg/segdata.h b/contrib/seg/segdata.h
index 3d6e3e3f24506979f5c40f0933abc3e5c1808eb9..7bc7c83dca309153e186b888da1efa25d1c9f1cf 100644 (file)
--- a/contrib/seg/segdata.h
+++ b/contrib/seg/segdata.h
@@ -14,12 +14,17 @@ typedef struct SEG
 /* in seg.c */
 extern int significant_digits(const char *s);
 
+/* for segscan.l and segparse.y */
+union YYSTYPE;
+typedef void *yyscan_t;
+
 /* in segscan.l */
-extern int seg_yylex(void);
+extern int seg_yylex(union YYSTYPE *yylval_param, yyscan_t yyscanner);
 extern void seg_yyerror(SEG *result, struct Node *escontext,
+                       yyscan_t yyscanner,
                        const char *message);
-extern void seg_scanner_init(const char *str);
-extern void seg_scanner_finish(void);
+extern void seg_scanner_init(const char *str, yyscan_t *yyscannerp);
+extern void seg_scanner_finish(yyscan_t yyscanner);
 
 /* in segparse.y */
-extern int seg_yyparse(SEG *result, struct Node *escontext);
+extern int seg_yyparse(SEG *result, struct Node *escontext, yyscan_t yyscanner);

diff --git a/contrib/seg/segparse.y b/contrib/seg/segparse.y
index 3e4aa31cada653b80956487edc032cc65eaea5d7..0358ddb182cb52af198e0c73a7523a52e4c1800d 100644 (file)
--- a/contrib/seg/segparse.y
+++ b/contrib/seg/segparse.y
@@ -14,10 +14,6 @@
 #include "segdata.h"
 #include "segparse.h"
 
-/* silence -Wmissing-variable-declarations */
-extern int seg_yychar;
-extern int seg_yynerrs;
-
 /*
  * Bison doesn't allocate anything that needs to live across parser calls,
  * so we can easily have it use palloc instead of malloc.  This prevents
@@ -35,6 +31,9 @@ static int sig_digits(const char *value);
 /* BISON Declarations */
 %parse-param {SEG *result}
 %parse-param {struct Node *escontext}
+%parse-param {yyscan_t yyscanner}
+%lex-param   {yyscan_t yyscanner}
+%pure-parser
 %expect 0
 %name-prefix="seg_yy"
 
@@ -72,6 +71,8 @@ range: boundary PLUMIN deviation
        result->u_sigd = Max(sig_digits(strbuf), Max($1.sigd, $3.sigd));
        result->l_ext = '\0';
        result->u_ext = '\0';
+
+       (void) yynerrs; /* suppress compiler warning */
    }
 
    | boundary RANGE boundary

diff --git a/contrib/seg/segscan.l b/contrib/seg/segscan.l
index 4ad529eccc468a016bb9566d3b10bfafa6c3f892..88d5487613ed3d069151b68bf4c4b5ab279b855f 100644 (file)
--- a/contrib/seg/segscan.l
+++ b/contrib/seg/segscan.l
@@ -6,12 +6,8 @@
 
 #include "nodes/miscnodes.h"
 
-/*
- * NB: include segparse.h only AFTER including segdata.h, because segdata.h
- * contains the definition for SEG.
- */
 #include "segdata.h"
-#include "segparse.h"
+#include "segparse.h"  /* must be after segdata.h for SEG */
 }
 
 %{
@@ -29,18 +25,19 @@ fprintf_to_ereport(const char *fmt, const char *msg)
 {
    ereport(ERROR, (errmsg_internal("%s", msg)));
 }
-
-/* Handles to the buffer that the lexer uses internally */
-static YY_BUFFER_STATE scanbufhandle;
-static char *scanbuf;
 %}
 
+%option reentrant
+%option bison-bridge
 %option 8bit
 %option never-interactive
 %option nodefault
 %option noinput
 %option nounput
 %option noyywrap
+%option noyyalloc
+%option noyyrealloc
+%option noyyfree
 %option warn
 %option prefix="seg_yy"
 
@@ -53,12 +50,12 @@ float        ({integer}|{real})([eE]{integer})?
 
 %%
 
-{range}      seg_yylval.text = yytext; return RANGE;
-{plumin}     seg_yylval.text = yytext; return PLUMIN;
-{float}      seg_yylval.text = yytext; return SEGFLOAT;
-\<           seg_yylval.text = "<"; return EXTENSION;
-\>           seg_yylval.text = ">"; return EXTENSION;
-\~           seg_yylval.text = "~"; return EXTENSION;
+{range}      yylval->text = yytext; return RANGE;
+{plumin}     yylval->text = yytext; return PLUMIN;
+{float}      yylval->text = yytext; return SEGFLOAT;
+\<           yylval->text = "<"; return EXTENSION;
+\>           yylval->text = ">"; return EXTENSION;
+\~           yylval->text = "~"; return EXTENSION;
 [ \t\n\r\f\v]+ /* discard spaces */
 .            return yytext[0]; /* alert parser of the garbage */
 
@@ -67,8 +64,10 @@ float        ({integer}|{real})([eE]{integer})?
 /* LCOV_EXCL_STOP */
 
 void
-seg_yyerror(SEG *result, struct Node *escontext, const char *message)
+seg_yyerror(SEG *result, struct Node *escontext, yyscan_t yyscanner, const char *message)
 {
+   struct yyguts_t * yyg = (struct yyguts_t *) yyscanner;  /* needed for yytext macro */
+
    /* if we already reported an error, don't overwrite it */
    if (SOFT_ERROR_OCCURRED(escontext))
        return;
@@ -96,25 +95,16 @@ seg_yyerror(SEG *result, struct Node *escontext, const char *message)
  * Called before any actual parsing is done
  */
 void
-seg_scanner_init(const char *str)
+seg_scanner_init(const char *str, yyscan_t *yyscannerp)
 {
-   Size    slen = strlen(str);
-
-   /*
-    * Might be left over after ereport()
-    */
-   if (YY_CURRENT_BUFFER)
-       yy_delete_buffer(YY_CURRENT_BUFFER);
-
-   /*
-    * Make a scan buffer with special termination needed by flex.
-    */
-   scanbuf = palloc(slen + 2);
-   memcpy(scanbuf, str, slen);
-   scanbuf[slen] = scanbuf[slen + 1] = YY_END_OF_BUFFER_CHAR;
-   scanbufhandle = yy_scan_buffer(scanbuf, slen + 2);
-
-   BEGIN(INITIAL);
+   yyscan_t yyscanner;
+
+   if (yylex_init(yyscannerp) != 0)
+       elog(ERROR, "yylex_init() failed: %m");
+
+   yyscanner = *yyscannerp;
+
+   yy_scan_string(str, yyscanner);
 }
 
 
@@ -122,8 +112,34 @@ seg_scanner_init(const char *str)
  * Called after parsing is done to clean up after seg_scanner_init()
  */
 void
-seg_scanner_finish(void)
+seg_scanner_finish(yyscan_t yyscanner)
+{
+   yylex_destroy(yyscanner);
+}
+
+/*
+ * Interface functions to make flex use palloc() instead of malloc().
+ * It'd be better to make these static, but flex insists otherwise.
+ */
+
+void *
+yyalloc(yy_size_t size, yyscan_t yyscanner)
+{
+   return palloc(size);
+}
+
+void *
+yyrealloc(void *ptr, yy_size_t size, yyscan_t yyscanner)
+{
+   if (ptr)
+       return repalloc(ptr, size);
+   else
+       return palloc(size);
+}
+
+void
+yyfree(void *ptr, yyscan_t yyscanner)
 {
-   yy_delete_buffer(scanbufhandle);
-   pfree(scanbuf);
+   if (ptr)
+       pfree(ptr);
 }