% Copyright 2012-2024, Alexander Shibakov
% This file is part of SPLinT
%
% SPLinT is free software: you can redistribute it and/or modify
% it under the terms of the GNU General Public License as published by
% the Free Software Foundation, either version 3 of the License, or
% (at your option) any later version.
%
% SPLinT is distributed in the hope that it will be useful,
% but WITHOUT ANY WARRANTY; without even the implied warranty of
% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
% GNU General Public License for more details.
%
% You should have received a copy of the GNU General Public License
% along with SPLinT.  If not, see <http://www.gnu.org/licenses/>.

% the implementation of (some part of) the \flex automata mechanisms;
% note that a few important features have been omitted and some that
% have been implemented may follow a (sometimes subtly) different
% semantics (see \unput and \yyless below);
% among the features that are missing are REJECT, ECHO and YYMORE;
% another important missing feature is the lack of trailing
% context matching; fixed length trailing context or fixed length
% match with variable length context can be easily emulated with
% regular rules and \yyless (or \unput); when both the match and
% the context are variable length, different automaton code is
% required; same code is required to implement REJECT; in addition,
% the action code for the actions matching trailing context rules
% must also be modified; while all of this may be implemented, it
% is better not to (ab)use trailing context rules.

\def\yylex{%
    \ifnum\yyg@yyinit=\z@           % if ( !yyg->yy_init ) {
        \yyg@yyinit=\@ne            %     yyg->yy_init = 1;
        \ifnum\yyg@yystart=\z@      %     if ( ! yyg->yy_start ) {
            \yyg@yystart=\@ne       %         yyg->yy_start = 1;
        \fi                         %     }
    \fi                             % }
    \yylwhile
}

\newif\ifyylessused
\newcount\yylesslastfmark
\newcount\yylesslastsmark
\newcount\yylesslastchar

\def\yylwhile{%
                                    % yy_cp = yyg->yy_c_buf_p;
                                    % *yy_cp = yyg->yy_hold_char;
                                    % yy_bp = yy_cp;
                                    %
    \yycurrentstate=\yyg@yystart    % yy_current_state = yyg->yy_start;
    \advance\yycurrentstate\YYATBOL % yy_current_state += YY_AT_BOL();
    % save the format and stash streams for \yyless
    \ifyylessused
        \yylesslastfmark\yyfmark
        \yylesslastsmark\yysmark
        \yylesslastchar\yytextlastchar
    \fi
    \yymatch
}

\newif\ifyytextbackup
\newif\iftraceflexbuffers

% an implementation of \yymatch with a low level command \yyreadinput
% that allows for a general input routine to be used at the beginning of the input 
% buffer (see an example of using \yyreadinput in \ldcleanyyeof macro in ldlex.w)

\def\yyresetstreams{%
    \iftraceflexbuffers\ferrmessage{will rescan: <\the\yytext@seen>}\fi
    \yytextseen{}\yytext@seen{}\yytextseenpure{}%
    \yytextbackupfalse
}

\def\yyreadinput#1#2{%
    \expandafter\yyre@dinput\expandafter{#2}{#1}%
}

\def\yyre@dinput#1#2{%
    #2#1%
}

\def\yyr@@dinput{%
    \ifyytextbackup
        \yybreak{\expandafter{\the\yytext@seen}{\yyresetstreams}}%
    \else
        \yybreak{{}{}}%
    \yycontinue
}

\def\yyr@@dinp@t#1#2{%
    #2\yyinput#1%
}

\def\yymatch{\yyreadinput{\yyr@@dinp@t}{\romannumeral0\yyr@@dinput}}

% \yym@tch is the return point from the low-level input routine

\def\yym@tch{%                                      do {
    \yyc=\fgetelemof{yyec}\at\yycp@\relax         %     yy_c = yy_ec[YY_SC_TO_UI(*yy_cp)];
    \yyact=\fgetelemof{yyaccept}\at\yycurrentstate\relax  
                                                  % ... ... reuse \yyact ( yy_act = yy_accept[yy_current_state] )
    \ifyyflexdebug\ferrmessage{match action: \the\yyact\space(yytext: \the\yytext@:^^Jyytextseen: \the\yytext@seen>\the\yybyte)}\fi
    \ifnum\yyact=\z@                              %     if ( yy_accept[yy_current_state] ) {
    \else                                         % 
        \ifyyflexdebug\ferrmessage{yym@tch: smark: \the\yysmarklast}\fi
        \yyg@yylastacceptingstate=\yycurrentstate %         yyg->yy_last_accepting_state = yy_current_state;
        \concat\yytext\yytextseen                 %         yyg->yy_last_accepting_cpos = yy_cp;
        \concat\yytext@\yytext@seen               % ... ...
        \concat\yytextpure\yytextseenpure         % ... ...
        \yytextlastchar=\yytextseenlastchar       %
        \yyfmark=\yyfmarklast                     % ... ...
        \yysmark=\yysmarklast                     % ... ...
        \yyfmark@accept=\yyfmark
        \yysmark@accept=\yysmark
        \yytextseen={}\yytext@seen={}\yytextseenpure{}%
    \fi                                           %     }
    \yyllwhile
}

\let\yyreturn\yym@tch

% \yyllwhile searches for an accepting of rejecting state

\def\yyllwhile{%                                             ... yyllwhile:
    \yyact=\fgetelemof{yybase}\at\yycurrentstate\relax     % ... ... reusing \yyact ( yy_act = yy_base[yy_current_state] )
    \advance\yyact\yyc                                     % ... ... ( yy_act = yy_act + yy_c )
    \yyact=\fgetelemof{yychk}\at\yyact\relax               % ... ... ( yy_act = yy_chk[yy_act] ( == yy_chk[yy_base[yy_current_state] + yy_c] ) )
    \ifnum\yyact=\yycurrentstate                           % while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) {
        \yybreak\yymatchtail                               % ... if ( yy_chk[yy_base[yy_current_state] + yy_c] == yy_current_state )
    \else                                                  % ...     goto yy_match_tail;
                                                           % ... else
        \yycurrentstate=\fgetelemof{yydef}\at\yycurrentstate\relax 
                                                           %     yy_current_state = (int) yy_def[yy_current_state];
        \ifnum\yycurrentstate>\YYMAXREALCHAR\relax         %     if ( yy_current_state >= sizeof( yy_accept ) ) {
            \yyc=\fgetelemof{yymeta}\at\yyc\relax          %         yy_c = yy_meta[(unsigned int) yy_c];
        \fi                                                %     }
        \yybreak\yyllwhile                                 % ... goto yyllwhile;
    \yycontinue                                              % }
}

\def\yymatchtail{%
    \yycurrentstate=\fgetelemof{yybase}\at\yycurrentstate\relax
    \advance\yycurrentstate\yyc                            %
    \yycurrentstate=\fgetelemof{yynxt}\at\yycurrentstate\relax
    \ifyyflexdebug\ferrmessage{yysubtext: (\the\yysubtext),^^Jyybyte: (\the\yybyte)}\fi
                                                           % yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c];
    \concat\yytextseen\yybyte                              % ++yy_cp;
    \concat\yytext@seen\yysubtext \yysubtext{}%            % ...
    \concat\yytext@seen\yybyte                             % ...
    \concat\yytextseenpure\yybytepure                      % ...
    \yytextseenlastchar=\yycp@                             %
    \yyfmarklast=\formatmarker                             % ...
    \yysmarklast=\stashmarker                              % ...
    \yyact=\fgetelemof{yybase}\at\yycurrentstate\relax     % ... reusing \yyact ( yy_act = yy_base[yy_current_state] )
    \ifyyflexdebug\ferrmessage{matching the rest: <\the\yytext@seen>}\fi
    \ifnum\yyact=\YYBASEMAXENTRY\relax                     % } while ( yy_base[yy_current_state] != max( yy_base ) );
        \ifyyflexdebug\ferrmessage{looking for the action}\fi
        \yybreak\yyfindaction                              % ... if ( yy_base[yy_current_state] == max( yy_base ) ) goto yy_find_action;
    \else                                                  % ... else goto yy_match;
        \ifyyflexdebug\ferrmessage{looking for more input}\fi
        \yybreak\yymatch                                  
    \yycontinue
}

\newif\ifyyflexdebug
\newif\iftracebadchars

\def\yyfindaction{%
    \yyact=\fgetelemof{yyaccept}\at\yycurrentstate\relax   % yy_act = yy_accept[yy_current_state];
    \ifnum\yyact=\z@                                       % if ( yy_act == 0 ) {
        \yytextbackuptrue                                  %     yy_cp = yyg->yy_last_accepting_cpos;
        \yycurrentstate=\yyg@yylastacceptingstate          %     yy_current_state = yyg->yy_last_accepting_state;
        \yyact=\fgetelemof{yyaccept}\at\yycurrentstate\relax%    yy_act = yy_accept[yy_current_state];
        \formatmarker=\yyfmark@accept                      % ...
        \stashmarker=\yysmark@accept                       % ...
        \ifyyflexdebug\ferrmessage{backup: \the\yytext@:\the\yytext@seen^^J^^J%
                                   from state: \the\yyg@yylastacceptingstate\space
                                   stashmarker: \the\stashmarker}\fi
    \else
        % note that at this point \yytextbackupfalse is already set
        \concat\yytext\yytextseen
        \concat\yytext@\yytext@seen
        \concat\yytextpure\yytextseenpure
        \yytextlastchar=\yytextseenlastchar
        \yyfmark=\yyfmarklast % == \formatmarker
        \yysmark=\yysmarklast % == \stashmarker
        \yytextseen{}\yytext@seen{}\yytextseenpure{}%
                                                           %
    \fi                                                    % }
    \YYDOBEFOREACTION                                      % YY_DO_BEFORE_ACTION;
    \ifyyflexdebug
        \ferrmessage{action: \the\yyact,^^J%
                     state: \the\yycurrentstate\space(\the\yytext@:\the\yytext@seen)%
        }%
    \fi
    \doaction
}

\newif\iftraceactioncode

\def\YYDOBEFOREACTION{%
}

% \yytext---current yytext
% \yytextseen---last accepted yytext
% |--\yytext--|--\yytextseen--|--remaining buffer--|
% after accepting state:
% [---------\yytext-----------]                    
% after backing up:
% |--\yytext--|--\yytextseen-----remaining buffer--|
% \yytextseen={}

%#define YY_DO_BEFORE_ACTION \
%	yyg->yytext_ptr = yy_bp; \
%	yyleng = (size_t) (yy_cp - yy_bp); \
%	yyg->yy_hold_char = *yy_cp; \
%	*yy_cp = '\0'; \
%	yyg->yy_c_buf_p = yy_cp;

\def\doaction{%
    \iftraceactioncode
        {%
            \expandafter\toksa
                \expandafter\expandafter\expandafter{\csname doflexaction\number\yyact\parsernamespace\endcsname}%
            \ferrmessage{action code (\the\yyact):^^J\the\toksa}%
        }%
    \fi
    \yydoactionswitch\yyact
}

%#define YY_STATE_EOF(state) (YY_END_OF_BUFFER + state + 1)

% as the macros above and below make a lot of assignments, 
% expandability was not a
% priority, however, keeping the macro namespace clean was;
% expandability is, of course a guaranteed way to do so, grouping is
% another; 

\def\yylexeofaction{%
    \ifyyflexdebug
        \ferrmessage{eof action: .\the\yytext@>>\the\yytext@seen(\the\yytext)}%
    \fi
    \yygetnextbuffer
    \ifYYEOBLASTMATCH
        \yygetpreviousstate
        \ifyyflexdebug\ferrmessage{found previous state: \the\yycurrentstatelocal\space 
                                   matched text: \the\yytext@>>\the\yytext@seen(\the\yybyte)}\fi
        \yycurrentstate\yycurrentstatelocal
        \let\yylextail\yyfind@ction
        \YYEOBLASTMATCHfalse
    \else
        \yyact\YYENDOFBUFFER\relax
        \advance\yyact\YYSTART
        \advance\yyact\@ne        % yy_act = YY_STATE_EOF(YY_START);
        \let\yylextail\doaction   % goto do_action;
        \concatl\yytext\yytext@seen \yytextbackuptrue % in case the scanner gets called again
        \yytoksempty{\yytext}%
            {\errmessage{internal error: matched nothing in eob state}}%
            {\yyisthiscsr\yytext\yyeof{}{\errmessage{internal error: matched more than a single eob: (\the\yytext)}}}%
        \ifyyflexdebug
            \ferrmessage{eob in user state: \the\yyact\space matched text: \the\yytext@(\the\yytext)\space
                         to read: \the\yytext@seen}%
        \fi
        \yytext{}\yytext@{}%
    \fi
}

\def\yygetnextbuffer{%
    \yytoksempty{\yytext}{\errmessage{internal error: empty matched text in eob state}}%
        {% this somewhat complicated definition is needed in case the last character is an ordinary 
         % space character
            \expandafter\yystartsinspace\expandafter{\the\yytext}{%
                \expandafter\expandafter\expandafter
                    \yystringempty % we have matched exactly one token (space)
                \expandafter\expandafter\expandafter{\expandafter\eattospace\the\yytext}%
                    {\YYEOBLASTMATCHfalse}{\YYEOBLASTMATCHtrue}%
            }{%
                \expandafter\expandafter\expandafter
                    \yystringempty % we have matched exactly one token
                \expandafter\expandafter\expandafter{\expandafter\eatone\the\yytext}%
                    {\YYEOBLASTMATCHfalse}{\YYEOBLASTMATCHtrue}%
            }%
        }%
}

\newif\ifYYEOBLASTMATCH

\def\yygetpreviousstate{%
    \yycurrentstatelocal=\yyg@yystart
    \advance\yycurrentstatelocal\YYATBOL
    \ifyyflexdebug\ferrmessage{state setup.: start(\the\yyg@yystart), bol(\number\YYATBOL)}\fi
    %
    \yytoksempty{\yytext}{\errmessage{internal error: empty matched text in eob state}}{}%
    % move the scanned \yyeof to the buffer of seen characters
    \expandafter\yysplitoff\expandafter.\the\yytext\yyeof\end\yytext
    \expandafter\yysplitoff\expandafter.\the\yytext@\yyeof\end\yytext@
    \appendlnx\yytextseen\yyeof
    \appendlnx\yytext@seen\yyeof \yytextbackuptrue
    \ifyyflexdebug\ferrmessage{searching for last match in text: \the\yytext@}\fi
    \yytoksempty{\yytext}{}{% yyg->yytext_ptr >= yyg->yy_c_buf_p /* do not enter the for loop */
        \expandafter\yygetpreviousstatefor\the\yytext@\splitoffend % at least one iteration of the for loop
    }%
}

\def\yysplitoff#1\yyeof#2\end#3{%
    \yystringempty{#2}{%
        \errmessage{internal error: something other than \noexpand\yyeof\space triggered eob action: <\the\yytext@>}%
    }{%
        #3\expandafter{\eatone#1}% to remove the `.' at the beginning
    }%
}

\def\yygetpreviousstatefor{%
    % save all the lexer variables affected by \yyinput
    % to be restored on exit from the `for' loop
    % i.e.~when the \splitoff token is read
    \let\currentyyreturn\yyreturn
    \yyfutureyytext\yytext@seen % to be reinserted in yyfind@action
    \yytext{}\yytext@{}\yytextpure{}%
    \yytextseen{}\yytext@seen{}\yytextseenpure{}%
    \formatmarker=\yyfmark@accept
    \stashmarker=\yysmark@accept
    \yyfmarklast=\formatmarker
    \yysmarklast=\stashmarker
    \let\yyreturn\yygetpreviousstatef@r
    \yyinput
}

\def\yygetpreviousstatef@r{%
    \let\default\yygetpreviousstatedefault
    \ifyyflexdebug\ferrmessage{just read: \the\yybyte (so far: \the\yytext@><\the\yytext@seen)}\fi % TODO: delete
    \switchon{\expandafter\getfirsttoken\expandafter{\the\yybyte}}\in\yygetpreviousstateswitch
}

\def\yygetpreviousstateswitch{%
    \yyeof {% scanned a NUL: currently NUL transitions are not implemented
            % (so one cannot use NUL's in the input) so this state is never used
        \yyclocal\YYECMAGIC\relax % the constant extracted from yy_get_next_state
        \edef\yycurrentstatelocal@prejam{\the\yycurrentstatelocal}% 
        \yygetpreviousstat@f@r
    }%
    \splitoffend {% yyg->yytext_ptr >= yyg->yy_c_buf_p /* exit the for loop */
        % this is not an actual character but a marker for the end of the `for' loop
        \yycp@=\YYENDOFBUFFERCHAR\relax % this is not necessary
        \yybyte{}\yybytepure{}%
%
        \let\yyreturn\currentyyreturn
    }%
    \endparse \endparseinput {%
        \errmessage{internal error.: reading past the end of the input buffer}%
    }%
}

\def\yygetpreviousstatedefault{%
    \yyclocal=\fgetelemof{yyec}\at\yycp@\relax           %     yy_c = yy_ec[YY_SC_TO_UI(*yy_cp)];
    \yytextseenlastchar=\yycp@
    \yygetpreviousstat@f@r
}

\def\yygetpreviousstat@f@r{%
    \ifnum\fgetelemof{yyaccept}\at\yycurrentstatelocal=\z@
                                                  %     if ( yy_accept[yy_current_state] ) {
        \concat\yytextseen\yybyte
        \concat\yytext@seen\yysubtext \yysubtext{}%
        \concat\yytext@seen\yybyte
        \concat\yytextseenpure\yybytepure
        \yybyte{}\yybytepure{}%
        \yyfmarklast=\formatmarker
        \yysmarklast=\stashmarker
    \else                                         % 
        \ifyyflexdebug
            \ferrmessage{possible accepting state.: \the\yycurrentstatelocal:%
                         \fgetelemof{yyaccept}\at\yycurrentstatelocal}%
        \fi
        \yyg@yylastacceptingstate=\yycurrentstatelocal
                                                  %         yyg->yy_last_accepting_state = yy_current_state;
        \concat\yytext\yytextseen                 %         yyg->yy_last_accepting_cpos = yy_cp; TODO:?????????
        \concat\yytext@\yytext@seen               % ...
        \concat\yytextpure\yytextseenpure         % ... 
        \yytextseen=\yybyte                       % ...
        \yytext@seen=\yysubtext \yysubtext{}%     % ...
        \concat\yytext@seen\yybyte \yybyte{}%     % ...
        \yytextseenpure=\yybytepure\yybytepure{}% %
        \yytextbackuptrue
        \yytextlastchar=\yytextseenlastchar       %
                                                  %
        \yyfmark=\yyfmarklast                     % ... ...
        \yysmark=\yysmarklast                     % ... ...
        \yyfmark@accept=\yyfmark
        \yysmark@accept=\yysmark
        \yyfmarklast=\formatmarker
        \yysmarklast=\stashmarker
    \fi                     
    \yygpswhile
}


\def\yygpswhile{%                                            ... yyllwhile:
    \yyact=\fgetelemof{yybase}\at\yycurrentstatelocal\relax% ... ... reusing \yyact ( yy_act = yy_base[yy_current_state] )
    \advance\yyact\yyclocal                                % ... ... ( yy_act = yy_act + yy_c )
    \ifnum\fgetelemof{yychk}\at\yyact=\yycurrentstatelocal % while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) {
        \yybreak\yygpsfornext                              % ... if ( yy_chk[yy_base[yy_current_state] + yy_c] == yy_current_state )
                                                           % ... ... exit the while loop
    \else                                                  % ...     return yy_current_state;
                                                           % ... else
        \yycurrentstatelocal=\fgetelemof{yydef}\at\yycurrentstatelocal\relax 
                                                           %     yy_current_state = (int) yy_def[yy_current_state];
        \ifnum\yycurrentstatelocal>\YYMAXREALCHAR\relax    %     if ( yy_current_state >= sizeof( yy_accept ) ) {
            \yyclocal=\fgetelemof{yymeta}\at\yyclocal\relax%         yy_c = yy_meta[(unsigned int) yy_c];
        \fi                                                %     }
        \yybreak\yygpswhile                                % ... goto yyllwhile;
    \yycontinue                                            % }
}

\newif\ifyyflexisjammed

\def\yygpsfornext{% this combines yy_get_previous_state() and yy_get_NUL_trans()
    \ifyyflexdebug\ferrmessage{starting in state.: \the\yycurrentstatelocal}\fi % TODO: delete
    \yycurrentstatelocal=\fgetelemof{yybase}\at\yycurrentstatelocal\relax
    \advance\yycurrentstatelocal\yyclocal
    \yycurrentstatelocal=\fgetelemof{yynxt}\at\yycurrentstatelocal\relax
    % yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c];
    % currently, we do not allow NULs in the input so the code below is not used
    % when NUL transitions are implemented, this code will come into play
    \ifnum\yycurrentstatelocal=\YYMAXREALCHAR\relax
        \yycurrentstatelocal=\yycurrentstatelocal@prejam\relax
        \ifyyflexdebug\ferrmessage{jammed, switching to \yycurrentstatelocal@prejam}\fi
        \yyflexisjammedtrue
    \else
        \yyflexisjammedfalse
    \fi
    %yy_is_jam = (yy_current_state == ...);
    %return yy_is_jam ? 0 : yy_current_state;
    \ifyyflexdebug\ferrmessage{switching to state.: \the\yycurrentstatelocal\space on char \the\yyclocal}\fi % TODO: delete
    \yyinput
}

\def\yyfind@ction{% we have to use this macro instead of the
                  % regular \yyfindaction because we use a stop token (\splitoffend) to terminate
                  % the `for' loop and are thus `one token behind' by the time this action is taken
                  % therefore we have to forego the `yy_cp actions' in the original \yyfindaction
    \yyact=\fgetelemof{yyaccept}\at\yycurrentstate\relax   % yy_act = yy_accept[yy_current_state];
    \ifnum\yyact=\z@                                       % if ( yy_act == 0 ) {
                                                           %     yy_cp = yyg->yy_last_accepting_cpos;
        \yycurrentstate=\yyg@yylastacceptingstate          %     yy_current_state = yyg->yy_last_accepting_state;
        \yyact=\fgetelemof{yyaccept}\at\yycurrentstate\relax%    yy_act = yy_accept[yy_current_state];
        \formatmarker=\yyfmark@accept                      % ...
        \stashmarker=\yysmark@accept                       % ...
        \ifyyflexdebug\ferrmessage{backup.: \the\yytext:\the\yytextseen}\fi
    \else
                                                           %
        \concat\yytext\yytextseen
        \concat\yytext@\yytext@seen
        \concat\yytextpure\yytextseenpure
        \concat\yytext\yybyte
        \concat\yytext@\yysubtext \yysubtext{}%
        \concat\yytext@\yybyte
        \concat\yytextpure\yybytepure
        \yybyte{}\yybytepure{}%
        \yytextseen{}\yytext@seen{}\yytextseenpure{}%
        \yytextlastchar=\yytextseenlastchar
        \yyfmark=\yyfmarklast
        \yysmark=\yysmarklast
                                                           %
    \fi                                                    % }
    \concat\yytext@seen\yyfutureyytext                     % ... reinsert the tokens seen by the lexer before 
                                                           % ... \yygetpreviousaction got rolling
    \yyfutureyytext{}%
    \yytextbackuptrue
    \YYDOBEFOREACTION                                      % YY_DO_BEFORE_ACTION;
    \ifyyflexdebug
        \ferrmessage{action.: \the\yyact, state: \the\yycurrentstate\space(\the\yytext) \parsernamespace}%
    \fi
    \doaction
}

\def\YYRULESETUP{%                  %#define YY_RULE_SETUP \
    \yytoksempty{\yytext}{}{%       %	if ( yyleng > 0 ) \
        \ifnum\n=\yytextlastchar    %		YY_CURRENT_BUFFER_LVALUE->yy_at_bol = (yytext[yyleng - 1] == '\n'); \
            \YYATBOL=\@ne           %	YY_USER_ACTION
        \else                       %#define YY_AT_BOL() (YY_CURRENT_BUFFER_LVALUE->yy_at_bol)
            \YYATBOL=\z@
            \ifyyflexdebug\ferrmessage{YYATBOL: 0}\fi
        \fi
    }%
}

%#define yy_set_bol(at_bol) \
%       { \
%       if ( ! YY_CURRENT_BUFFER ){\
%        yyensure_buffer_stack (); \
%               YY_CURRENT_BUFFER_LVALUE =    \
%            yy_create_buffer(yyin,YY_BUF_SIZE ); \
%       } \
%       YY_CURRENT_BUFFER_LVALUE->yy_at_bol = at_bol; \
%       }

\def\yysetbol#1{\YYATBOL=#1\relax} % we do not switch or create buffers

%#define BEGIN yyg->yy_start = 1 + 2 *
% should be:
%#define BEGIN(state) yyg->yy_start = 1 + 2 * ( state )

\newif\iftracestates

\def\yylexstate#1{\csname flexstate\parsernamespace #1\endcsname}

\def\yyBEGIN#1{%
    \yyg@yystart
    \expandafter\ifx\csname flexstate\parsernamespace #1\endcsname\relax
        #1 \iftracestates\ferrmessage{now in state: #1}\fi
    \else
        \csname flexstate\parsernamespace #1\endcsname\relax
        \iftracestates\ferrmessage{now in state: #1 (\number\csname flexstate\parsernamespace #1\endcsname)}\fi
    \fi
    \multiply\yyg@yystart\tw@
    \advance\yyg@yystart\@ne
}

\def\yyBEGINr#1{%
    \yyg@yystart#1%
    \iftracestates\ferrmessage{new state: \the\yyg@yystart}\fi
    \multiply\yyg@yystart\tw@
    \advance\yyg@yystart\@ne
}

%#define YY_START ((yyg->yy_start - 1) / 2)

\def\YYSTART{%
    \expandafter\xdivbytwo\expandafter{\number
    \expandafter\xdecrement\expandafter{\number\yyg@yystart}}
}

\def\yypushstate#1{%
    \expandafter\yypush\expandafter{\number\YYSTART}\on\yystatestack
    \yyBEGIN{#1}%
}

\def\yypopstate{%
    \yypop\yystatestack\into{\expandafter\yyBEGIN\romannumeral0}%
    \iftracestates\ferrmessage{new state (* 2 + 1 scrambled): \the\yyg@yystart}\fi
}

\def\yytopstate#1{%
    \yyreadstack\yystatestack\at\z@\to#1%
}

\let\yylexcontinue\yylwhile % complete the while loop (requires \yylexnext)
\def\yylexcontinue{\yytext{}\yytextpure{}\yytext@{}\yylwhile} % complete the while loop

% \yylextail is the return point from the lexer

\def\yylexreturnbootstrap#1{%
    \yytext{}\yytext@{}\yytextpure{}%
    \expandafter\ifx\csname token\parsernamespace #1\endcsname\relax
                             % this token value is undefined
        \let\yylextail\yylex % so lex the next token
    \else
        \yychar\csname token\parsernamespace #1\endcsname\relax
        \let\yylextail\yyparsetail
    \fi
}

\def\yylexreturnregular#1{%
    \yychar\csname token\parsernamespace #1\endcsname\relax
    \yytext{}\yytext@{}\yytextpure{}\let\yylextail\yyparsetail
}

\def\yylexreturnval#1{% return value (yytext) only
    \yychar\csname token\parsernamespace #1\endcsname\relax
    {\edef\next{\yylval{{\the\yytext}{\the\yytextpure}}}\expandafter}\next
    \yytext{}\yytext@{}\yytextpure{}\let\yylextail\yyparsetail
}

\def\yylexreturnptr#1{% return stream pointers only
    {\edef\next{\yylval{{\the\yyfmark}{\the\yysmark}}}\expandafter}\next
    \yylexreturn{#1}%
}

\def\yylexreturntext{\yylexreturnptr{\the\yytextpure}}

\def\yylexreturnsym#1{% return the value (yytext) followed by the pointers
    \yychar\csname token\parsernamespace #1\endcsname\relax
    {\edef\next{\yylval{{\the\yytext}{\the\yytextpure}{\the\yyfmark}{\the\yysmark}}}\expandafter}\next
    \yytext{}\yytext@{}\yytextpure{}\let\yylextail\yyparsetail
}

\def\yylexreturnraw#1{% return the character, pointers as the value
    {\edef\next{\yylval{{\the\yyfmark}{\the\yysmark}}}\expandafter}\next
    \yychar`#1\relax
    \yytext{}\yytext@{}\yytextpure{}\let\yylextail\yyparsetail
}

\def\yylexreturnchar{%
    \expandafter\yylexreturnraw\the\yytextpure
}

\def\yylexreturnxchar#1{% return the numeric value, pointers as the value
    {\edef\next{\yylval{{\the\yyfmark}{\the\yysmark}}}\expandafter}\next
    \yychar#1\relax
    \yytext{}\yytext@{}\yytextpure{}\let\yylextail\yyparsetail
}

\def\yylexnext{\yytext{}\yytext@{}\yytextpure{}} % use this with a trivial \yylexcontinue
\let\yylexnext\empty

\def\yyterminate{\yychar\z@\yylval{}\yytext{}\yytext@{}\yytextpure{}\let\yylextail\yyparsetail}

\def\yyerrterminate{%
    \expandafter\ifx\csname token\parsernamespace $undefined\endcsname\relax
        \yybreak{\yylexreturn{"invalid token"}}%
    \else
        \yybreak{\yylexreturn{$undefined}}%$
    \yycontinue
}
    
\def\yyfatal#1{\yycomplain{#1}\yyerrterminate}

\def\yywarn#1{\yycomplain{#1}\yylexnext}

% \yyless is slow and the macros below have not been tested in their current
% form: avoid them

\def\yyless#1{%
    \yyless@backup
    \ifnum#1=\z@
        \yyfmark=\formatmarker
        \yysmark=\stashmarker
        \ROLLBACKCURRENTTOKEN
    \else
        \let\oldyyreturn\yyreturn
        \edef\yyreturn{\noexpand\yyskipnchars{\number#1}}%
        \expandafter\yyinput\the\yytext@\yyeof
    \fi
}

\def\yyless@backup{%
    \formatmarker\yylesslastfmark
    \stashmarker\yylesslastsmark
}

\def\yyskipnchars#1{%
    \ifnum\yycp@=\YYENDOFBUFFERCHAR % read \yyeof
        \yycomplain{yyless buffer overflow: #1 characters too many}%
    \else
        \ifnum#1=\@ne % skipped the required number of tokens
            \yybreak@\yyl@ss
        \else
            \edef\yyreturn{\noexpand\yyskipnchars{\xdecrement{#1}}}%
            \yybreak@\yyinput
        \fi
    \yycontinue
}

\def\yyl@ss#1\yyeof{%
    \yyfmark=\formatmarker
    \yysmark=\stashmarker
    \unput{#1}%
    \let\yyreturn\oldyyreturn
}

% \unput has a slightly different semantics from \flex's unput() since
% it puts back an arbitrary string rather than a character; it clobbers
% the values of \yytext, \yytext@, and \yytextpure so these must be saved before
% \unput is used

\def\unput#1{%
    \yytext@{#1}%
    \concatl\yytext@\yytext@seen
    \yytext{}\yytext@{}\yytextpure{}%
%    \yystringempty{#1}{}{\yytextbackuptrue}% TODO: just set \yytexbackuptrue
    \yytextbackuptrue
}

%#define ROLLBACK_CURRENT_TOKEN                                  \
%  do {                                                          \
%    scanner_cursor.column -= mbsnwidth (yytext, yyleng, 0);     \
%    yyless (0);                                                 \
%  } while (0)

\def\ROLLBACKCURRENTTOKEN{% this does not reset the streams
    \concatl\yytext\yytextseen
    \concatl\yytext\yytext@seen % do not rollback collected streams
    \yytext{}\yytext@{}\yytextpure{}%
    \yytextbackuptrue
}