C Tokenizer (and it returns empty too when fields are missing. yay!)

问题

See also: Is this a good substr() for C?

strtok() and friends skip over empty fields, and I do not know how to tell it not to skip but rather return empty in such cases.

Similar behavior from most tokenizers I could see, and don't even get me started on sscanf() (but then it never said it would work on empty fields to begin with).

I have been on a roll and feeling sleepy as well, so here it goes for review:

char* substr(const char* text, int nStartingPos, int nRun)
{
    char* emptyString = strdup(""); /* C'mon! This cannot fail */

    if(text == NULL) return emptyString;

    int textLen = strlen(text);

    --nStartingPos;

    if((nStartingPos < 0) || (nRun <= 0) || (textLen == 0) || (textLen < nStartingPos)) return emptyString;

    char* returnString = (char *)calloc((1 + nRun), sizeof(char));

    if(returnString == NULL) return emptyString;

    strncat(returnString, (nStartingPos + text), nRun);

    /* We do not need emptyString anymore from this point onwards */
    free(emptyString);
    emptyString = NULL;

    return returnString;
}

int TokenizeC(const char* text, char delim, char ***output)
{
    if((*output) != NULL) return -1; /* I will allocate my own storage */

    int nCountOfDelimiters = 0;
    int dx = 0;
    int nCountOfElements = 0;
    int textLen = strlen(text);

    for(; dx < textLen; ++dx)
    {
        if((text[dx] == delim) && (dx != (textLen - 1))) ++nCountOfDelimiters;
        /* trailing delimiter need not be counted separately as we are */
        /* incrementing the count always by 1 */
    }

    /*
    We will have as many array elements as nCountOfDelimiters + 1
    Tokenizing an empty string should return a single token that would
    be empty (Is this not how most libraries behave? Or should it return NULL?)
    */

    (*output) = (char **)malloc((1 + nCountOfDelimiters) * sizeof(char *));

    for(dx = 0; dx < textLen; dx++)
    {
        int nStartOfString = (1 + dx);

        //printf("\n[! 1]dx = %d, nStartOfString = %d", dx, nStartOfString);

        /* Get the run between delimiters */
        while((dx < textLen) && (text[dx] != delim)) dx++;

        //printf("\n[! 2]dx = %d, nStartOfString = %d", dx, nStartOfString);

        (*output)[nCountOfElements] = (1 + dx - nStartOfString) ? substr(text, nStartOfString, (1 + dx - nStartOfString)) : strdup("");

        //printf("\n[!]substr(text, %d, %d) => '%s'", nStartOfString, (1 + dx - nStartOfString), (*output)[nCountOfElements]);

        if(NULL == (*output)[nCountOfElements])
        {
            // Woops! Undo all
            // TODO: How to test this scenario?!

            for(; nCountOfElements >= 0; --nCountOfElements)
            {
                if((*output)[nCountOfElements] != NULL) free((*output)[nCountOfElements]);
                (*output)[nCountOfElements] = NULL;
            }

            return -2; 
        }

        ++nCountOfElements;
    }

    return nCountOfElements; /* Return the number of tokens if sucessful */
}

void reclaim2D(char ***store, unsigned int itemCount)
{
    for (int x = 0; itemCount < itemCount; ++x)
    {
        if((*store)[x] != NULL) free((*store)[x]);
        (*store)[x] = NULL;
    }

    if((*store) != NULL) free((*store));
    (*store) = NULL;
}

Here's the driver:

int main()
{
    // Trailing '-' scenarios not giving correct count of elements
    // (off by 1 for the last element that should come as empty)

    const char *text = "1-2-3-6-7-8-9-10-11-", delim = '-'; // 10 elements

    char **output = NULL;

    int c = TokenizeC(text, delim, &output);

    printf("\n\n[*]%d", c);

    for (int x = 0; x < c; ++x)
    {
        printf("\n[main]'%s'", output[x]); //Expected : 1-2-3-6-7-8-9-10-11-<empty>
    }

    reclaim2D(&output, c);

    text = "12-3-6-7-8-9-10-11";  // 8 elements

    c = TokenizeC(text, delim, &output);

    printf("\n\n[*]%d", c);

    for(int x = 0; x < c; ++x)
    {
        printf("\n[main]'%s'", output[x]); //Expected : 12-3-6-7-8-9-10-11
    }

    reclaim2D(&output, c);

    text = "-----2--4--6-7100000000-8-9-10-11-100000000-";  // 17 elements

    c = TokenizeC(text, delim, &output);

    printf("\n\n[*]%d", c);

    for(int x = 0; x < c; ++x)
    {
        printf("\n[main]'%s'", output[x]);
        //Expected <empty>-<empty>-<empty>-<empty>
        // -<empty>-2-<empty>-4-<empty>-6-7100000000
        // -8-9-10-11-100000000-<empty>
    }

    reclaim2D(&output, c);

    text = "-----2--4--6-7100000000-8-9-10-11-100000000";  // 16 elements

    c = TokenizeC(text, delim, &output);

    printf("\n\n[*]%d", c);

    for(int x = 0; x < c; ++x)
    {
        printf("\n[main]'%s'", output[x]);
        //Expected : <empty>-<empty>-<empty>-<empty>-<empty>
        //-2-<empty>-4-<empty>-6-7100000000-8-9-10-11-100000000
    }

    reclaim2D(&output, c);

    return 0;
}

Yes, you noticed it right; it works now only for a single delimiter, but of course, we have this off by one bug to attend to.

Outputs:

[*]9
[main]'1'
[main]'2'
[main]'3'
[main]'6'
[main]'7'
[main]'8'
[main]'9'
[main]'10'
[main]'11'

[*]8
[main]'12'
[main]'3'
[main]'6'
[main]'7'
[main]'8'
[main]'9'
[main]'10'
[main]'11'

[*]16
[main]''
[main]''
[main]''
[main]''
[main]''
[main]'2'
[main]''
[main]'4'
[main]''
[main]'6'
[main]'7100000000'
[main]'8'
[main]'9'
[main]'10'
[main]'11'
[main]'100000000'

[*]16
[main]''
[main]''
[main]''
[main]''
[main]''
[main]'2'
[main]''
[main]'4'
[main]''
[main]'6'
[main]'7100000000'
[main]'8'
[main]'9'
[main]'10'
[main]'11'
[main]'100000000'

I am also making this a wiki because I saw many similar requests on the net.

回答1:

On some system, there is a function called strsep(). And you can find its source code by google. For example, http://www.google.com/codesearch/p?hl=zh-TW#XAzRy8oK4zA/libc/string/strsep.c&q=strsep

来源：https://stackoverflow.com/questions/874161/c-tokenizer-and-it-returns-empty-too-when-fields-are-missing-yay

标签

substring

tokenize

substr