tokens.c

Go to the documentation of this file.

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "cpp.h"

static char wbuf[2*OBS];
static char *wbp = wbuf;

/*
 * 1 for tokens that don't need whitespace when they get inserted
 * by macro expansion
 */
static const char wstab[] = {
    0,  /* END */
    0,  /* UNCLASS */
    0,  /* NAME */
    0,  /* NUMBER */
    0,  /* STRING */
    0,  /* CCON */
    1,  /* NL */
    0,  /* WS */
    0,  /* DSHARP */
    0,  /* EQ */
    0,  /* NEQ */
    0,  /* LEQ */
    0,  /* GEQ */
    0,  /* LSH */
    0,  /* RSH */
    0,  /* LAND */
    0,  /* LOR */
    0,  /* PPLUS */
    0,  /* MMINUS */
    0,  /* ARROW */
    1,  /* SBRA */
    1,  /* SKET */
    1,  /* LP */
    1,  /* RP */
    0,  /* DOT */
    0,  /* AND */
    0,  /* STAR */
    0,  /* PLUS */
    0,  /* MINUS */
    0,  /* TILDE */
    0,  /* NOT */
    0,  /* SLASH */
    0,  /* PCT */
    0,  /* LT */
    0,  /* GT */
    0,  /* CIRC */
    0,  /* OR */
    0,  /* QUEST */
    0,  /* COLON */
    0,  /* ASGN */
    1,  /* COMMA */
    0,  /* SHARP */
    1,  /* SEMIC */
    1,  /* CBRA */
    1,  /* CKET */
    0,  /* ASPLUS */
    0,  /* ASMINUS */
    0,  /* ASSTAR */
    0,  /* ASSLASH */
    0,  /* ASPCT */
    0,  /* ASCIRC */
    0,  /* ASLSH */
    0,  /* ASRSH */
    0,  /* ASOR */
    0,  /* ASAND */
    0,  /* ELLIPS */
    0,  /* DSHARP1 */
    0,  /* NAME1 */
    0,  /* DEFINED */
    0,  /* UMINUS */
};

void
maketokenrow(int size, Tokenrow *trp)
{
    trp->max = size;
    if (size>0)
        trp->bp = (Token *)domalloc(size*sizeof(Token));
    else
        trp->bp = NULL;
    trp->tp = trp->bp;
    trp->lp = trp->bp;
}

Token *
growtokenrow(Tokenrow *trp)
{
    int ncur = trp->tp - trp->bp;
    int nlast = trp->lp - trp->bp;

    trp->max = 3*trp->max/2 + 1;
    trp->bp = (Token *)realloc(trp->bp, trp->max*sizeof(Token));
    if (trp->bp == NULL)
        error(FATAL, "Out of memory from realloc");
    trp->lp = &trp->bp[nlast];
    trp->tp = &trp->bp[ncur];
    return trp->lp;
}

/*
 * Compare a row of tokens, ignoring the content of WS; return !=0 if different
 */
int
comparetokens(Tokenrow *tr1, Tokenrow *tr2)
{
    Token *tp1, *tp2;

    tp1 = tr1->tp;
    tp2 = tr2->tp;
    if (tr1->lp-tp1 != tr2->lp-tp2)
        return 1;
    for (; tp1<tr1->lp ; tp1++, tp2++) {
        if (tp1->type != tp2->type
         || (tp1->wslen==0) != (tp2->wslen==0)
         || tp1->len != tp2->len
         || strncmp((char*)tp1->t, (char*)tp2->t, tp1->len)!=0)
            return 1;
    }
    return 0;
}

/*
 * replace ntok tokens starting at dtr->tp with the contents of str.
 * tp ends up pointing just beyond the replacement.
 * Canonical whitespace is assured on each side.
 */
void
insertrow(Tokenrow *dtr, int ntok, Tokenrow *str)
{
    int nrtok = rowlen(str);

    dtr->tp += ntok;
    adjustrow(dtr, nrtok-ntok);
    dtr->tp -= ntok;
    movetokenrow(dtr, str);
    makespace(dtr);
    dtr->tp += nrtok;
    makespace(dtr);
}

/*
 * make sure there is WS before trp->tp, if tokens might merge in the output
 */
void
makespace(Tokenrow *trp)
{
    uchar *tt;
    Token *tp = trp->tp;

    if (tp >= trp->lp)
        return;
    if (tp->wslen) {
        if (tp->flag&XPWS
         && (wstab[tp->type] || trp->tp>trp->bp && wstab[(tp-1)->type])) {
            tp->wslen = 0;
            return;
        }
        tp->t[-1] = ' ';
        return;
    }
    if (wstab[tp->type] || trp->tp>trp->bp && wstab[(tp-1)->type])
        return;
    tt = newstring(tp->t, tp->len, 1);
    *tt++ = ' ';
    tp->t = tt;
    tp->wslen = 1;
    tp->flag |= XPWS;
}

/*
 * Copy an entire tokenrow into another, at tp.
 * It is assumed that there is enough space.
 *  Not strictly conforming.
 */
void
movetokenrow(Tokenrow *dtr, Tokenrow *str)
{
    int nby;

    /* nby = sizeof(Token) * (str->lp - str->bp); */
    nby = (char *)str->lp - (char *)str->bp;
    memmove(dtr->tp, str->bp, nby);
}

/*
 * Move the tokens in a row, starting at tr->tp, rightward by nt tokens;
 * nt may be negative (left move).
 * The row may need to be grown.
 * Non-strictly conforming because of the (char *), but easily fixed
 */
void
adjustrow(Tokenrow *trp, int nt)
{
    int nby, size;

    if (nt==0)
        return;
    size = (trp->lp - trp->bp) + nt;
    while (size > trp->max)
        growtokenrow(trp);
    /* nby = sizeof(Token) * (trp->lp - trp->tp); */
    nby = (char *)trp->lp - (char *)trp->tp;
    if (nby)
        memmove(trp->tp+nt, trp->tp, nby);
    trp->lp += nt;
}

/*
 * Copy a row of tokens into the destination holder, allocating
 * the space for the contents.  Return the destination.
 */
Tokenrow *
copytokenrow(Tokenrow *dtr, Tokenrow *str)
{
    int len = rowlen(str);

    maketokenrow(len, dtr);
    movetokenrow(dtr, str);
    dtr->lp += len;
    return dtr;
}

/*
 * Produce a copy of a row of tokens.  Start at trp->tp.
 * The value strings are copied as well.  The first token
 * has WS available.
 */
Tokenrow *
normtokenrow(Tokenrow *trp)
{
    Token *tp;
    Tokenrow *ntrp = new(Tokenrow);
    int len;

    len = trp->lp - trp->tp;
    if (len<=0)
        len = 1;
    maketokenrow(len, ntrp);
    for (tp=trp->tp; tp < trp->lp; tp++) {
        *ntrp->lp = *tp;
        if (tp->len) {
            ntrp->lp->t = newstring(tp->t, tp->len, 1);
            *ntrp->lp->t++ = ' ';
            if (tp->wslen)
                ntrp->lp->wslen = 1;
        }
        ntrp->lp++;
    }
    if (ntrp->lp > ntrp->bp)
        ntrp->bp->wslen = 0;
    return ntrp;
}

/*
 * Debugging
 */
void
peektokens(Tokenrow *trp, char *str)
{
    Token *tp;

    tp = trp->tp;
    flushout();
    if (str)
        fprintf(stderr, "%s ", str);
    if (tp<trp->bp || tp>trp->lp)
        fprintf(stderr, "(tp offset %d) ", tp-trp->bp);
    for (tp=trp->bp; tp<trp->lp && tp<trp->bp+32; tp++) {
        if (tp->type!=NL) {
            int c = tp->t[tp->len];
            tp->t[tp->len] = 0;
            fprintf(stderr, "%s", tp->t);
            tp->t[tp->len] = c;
        }
        if (tp->type==NAME) {
            fprintf(stderr, tp==trp->tp?"{*":"{");
            prhideset(tp->hideset);
            fprintf(stderr, "} ");
        } else
            fprintf(stderr, tp==trp->tp?"{%x*} ":"{%x} ", tp->type);
    }
    fprintf(stderr, "\n");
    fflush(stderr);
}

void
puttokens(Tokenrow *trp)
{
    Token *tp;
    int len;
    uchar *p;

    if (verbose)
        peektokens(trp, "");
    tp = trp->bp;
    for (; tp<trp->lp; tp++) {
        len = tp->len+tp->wslen;
        p = tp->t-tp->wslen;
        while (tp<trp->lp-1 && p+len == (tp+1)->t - (tp+1)->wslen) {
            tp++;
            len += tp->wslen+tp->len;
        }
        if (len>OBS/2) {        /* handle giant token */
            if (wbp > wbuf)
                write(1, wbuf, wbp-wbuf);
            write(1, (char *)p, len);
            wbp = wbuf;
        } else {    
            memcpy(wbp, p, len);
            wbp += len;
        }
        if (wbp >= &wbuf[OBS]) {
            write(1, wbuf, OBS);
            if (wbp > &wbuf[OBS])
                memcpy(wbuf, wbuf+OBS, wbp - &wbuf[OBS]);
            wbp -= OBS;
        }
    }
    trp->tp = tp;
    if (cursource->fd==0)
        flushout();
}

void
flushout(void)
{
    if (wbp>wbuf) {
        write(1, wbuf, wbp-wbuf);
        wbp = wbuf;
    }
}

/*
 * turn a row into just a newline
 */
void
setempty(Tokenrow *trp)
{
    trp->tp = trp->bp;
    trp->lp = trp->bp+1;
    *trp->bp = nltoken;
}

/*
 * generate a number
 */
char *
outnum(char *p, int n)
{
    if (n>=10)
        p = outnum(p, n/10);
    *p++ = n%10 + '0';
    return p;
}

/*
 * allocate and initialize a new string from s, of length l, at offset o
 * Null terminated.
 */
uchar *
newstring(uchar *s, int l, int o)
{
    uchar *ns = (uchar *)domalloc(l+o+1);

    ns[l+o] = '\0';
    return (uchar*)strncpy((char*)ns+o, (char*)s, l) - o;
}

---