How to split a string, but also keep the delimiters?

Question

I have a multiline string which is delimited by a set of different delimiters:

(Text1)(DelimiterA)(Text2)(DelimiterC)(Text3)(DelimiterB)(Text4)

Answer 1

import java.util.regex.*;
import java.util.LinkedList;

public class Splitter {
    private static final Pattern DEFAULT_PATTERN = Pattern.compile("\\s+");

    private Pattern pattern;
    private boolean keep_delimiters;

    public Splitter(Pattern pattern, boolean keep_delimiters) {
        this.pattern = pattern;
        this.keep_delimiters = keep_delimiters;
    }
    public Splitter(String pattern, boolean keep_delimiters) {
        this(Pattern.compile(pattern==null?"":pattern), keep_delimiters);
    }
    public Splitter(Pattern pattern) { this(pattern, true); }
    public Splitter(String pattern) { this(pattern, true); }
    public Splitter(boolean keep_delimiters) { this(DEFAULT_PATTERN, keep_delimiters); }
    public Splitter() { this(DEFAULT_PATTERN); }

    public String[] split(String text) {
        if (text == null) {
            text = "";
        }

        int last_match = 0;
        LinkedList<String> splitted = new LinkedList<String>();

        Matcher m = this.pattern.matcher(text);

        while (m.find()) {

            splitted.add(text.substring(last_match,m.start()));

            if (this.keep_delimiters) {
                splitted.add(m.group());
            }

            last_match = m.end();
        }

        splitted.add(text.substring(last_match));

        return splitted.toArray(new String[splitted.size()]);
    }

    public static void main(String[] argv) {
        if (argv.length != 2) {
            System.err.println("Syntax: java Splitter <pattern> <text>");
            return;
        }

        Pattern pattern = null;
        try {
            pattern = Pattern.compile(argv[0]);
        }
        catch (PatternSyntaxException e) {
            System.err.println(e);
            return;
        }

        Splitter splitter = new Splitter(pattern);

        String text = argv[1];
        int counter = 1;
        for (String part : splitter.split(text)) {
            System.out.printf("Part %d: \"%s\"\n", counter++, part);
        }
    }
}

/*
    Example:
    > java Splitter "\W+" "Hello World!"
    Part 1: "Hello"
    Part 2: " "
    Part 3: "World"
    Part 4: "!"
    Part 5: ""
*/

I don't really like the other way, where you get an empty element in front and back. A delimiter is usually not at the beginning or at the end of the string, thus you most often end up wasting two good array slots.

Edit: Fixed limit cases. Commented source with test cases can be found here: http://snippets.dzone.com/posts/show/6453

Answer 2

One of the subtleties in this question involves the "leading delimiter" question: if you are going to have a combined array of tokens and delimiters you have to know whether it starts with a token or a delimiter. You could of course just assume that a leading delim should be discarded but this seems an unjustified assumption. You might also want to know whether you have a trailing delim or not. This sets two boolean flags accordingly.

Written in Groovy but a Java version should be fairly obvious:

            String tokenRegex = /[\p{L}\p{N}]+/ // a String in Groovy, Unicode alphanumeric
            def finder = phraseForTokenising =~ tokenRegex
            // NB in Groovy the variable 'finder' is then of class java.util.regex.Matcher
            def finderIt = finder.iterator() // extra method added to Matcher by Groovy magic
            int start = 0
            boolean leadingDelim, trailingDelim
            def combinedTokensAndDelims = [] // create an array in Groovy

            while( finderIt.hasNext() )
            {
                def token = finderIt.next()
                int finderStart = finder.start()
                String delim = phraseForTokenising[ start  .. finderStart - 1 ]
                // Groovy: above gets slice of String/array
                if( start == 0 ) leadingDelim = finderStart != 0
                if( start > 0 || leadingDelim ) combinedTokensAndDelims << delim
                combinedTokensAndDelims << token // add element to end of array
                start = finder.end()
            }
            // start == 0 indicates no tokens found
            if( start > 0 ) {
                // finish by seeing whether there is a trailing delim
                trailingDelim = start < phraseForTokenising.length()
                if( trailingDelim ) combinedTokensAndDelims << phraseForTokenising[ start .. -1 ]

                println( "leading delim? $leadingDelim, trailing delim? $trailingDelim, combined array:\n $combinedTokensAndDelims" )

            }

Answer 3

You want to use lookarounds, and split on zero-width matches. Here are some examples:

public class SplitNDump {
    static void dump(String[] arr) {
        for (String s : arr) {
            System.out.format("[%s]", s);
        }
        System.out.println();
    }
    public static void main(String[] args) {
        dump("1,234,567,890".split(","));
        // "[1][234][567][890]"
        dump("1,234,567,890".split("(?=,)"));   
        // "[1][,234][,567][,890]"
        dump("1,234,567,890".split("(?<=,)"));  
        // "[1,][234,][567,][890]"
        dump("1,234,567,890".split("(?<=,)|(?=,)"));
        // "[1][,][234][,][567][,][890]"

        dump(":a:bb::c:".split("(?=:)|(?<=:)"));
        // "[][:][a][:][bb][:][:][c][:]"
        dump(":a:bb::c:".split("(?=(?!^):)|(?<=:)"));
        // "[:][a][:][bb][:][:][c][:]"
        dump(":::a::::b  b::c:".split("(?=(?!^):)(?<!:)|(?!:)(?<=:)"));
        // "[:::][a][::::][b  b][::][c][:]"
        dump("a,bb:::c  d..e".split("(?!^)\\b"));
        // "[a][,][bb][:::][c][  ][d][..][e]"

        dump("ArrayIndexOutOfBoundsException".split("(?<=[a-z])(?=[A-Z])"));
        // "[Array][Index][Out][Of][Bounds][Exception]"
        dump("1234567890".split("(?<=\\G.{4})"));   
        // "[1234][5678][90]"

        // Split at the end of each run of letter
        dump("Boooyaaaah! Yippieeee!!".split("(?<=(?=(.)\\1(?!\\1))..)"));
        // "[Booo][yaaaa][h! Yipp][ieeee][!!]"
    }
}

And yes, that is triply-nested assertion there in the last pattern.

Related questions

• Java split is eating my characters.
• Can you use zero-width matching regex in String split?
• How do I convert CamelCase into human-readable names in Java?
• Backreferences in lookbehind

See also

• regular-expressions.info/Lookarounds

Answer 4

I like the idea of StringTokenizer because it is Enumerable.
But it is also obsolete, and replace by String.split which return a boring String[] (and does not includes the delimiters).

So I implemented a StringTokenizerEx which is an Iterable, and which takes a true regexp to split a string.

A true regexp means it is not a 'Character sequence' repeated to form the delimiter:
'o' will only match 'o', and split 'ooo' into three delimiter, with two empty string inside:

[o], '', [o], '', [o]

But the regexp o+ will return the expected result when splitting "aooob"

[], 'a', [ooo], 'b', []

To use this StringTokenizerEx:

final StringTokenizerEx aStringTokenizerEx = new StringTokenizerEx("boo:and:foo", "o+");
final String firstDelimiter = aStringTokenizerEx.getDelimiter();
for(String aString: aStringTokenizerEx )
{
    // uses the split String detected and memorized in 'aString'
    final nextDelimiter = aStringTokenizerEx.getDelimiter();
}

The code of this class is available at DZone Snippets.

As usual for a code-challenge response (one self-contained class with test cases included), copy-paste it (in a 'src/test' directory) and run it. Its main() method illustrates the different usages.

---

Note: (late 2009 edit)

The article Final Thoughts: Java Puzzler: Splitting Hairs does a good work explaning the bizarre behavior in String.split().
Josh Bloch even commented in response to that article:

Yes, this is a pain. FWIW, it was done for a very good reason: compatibility with Perl.
The guy who did it is Mike "madbot" McCloskey, who now works with us at Google. Mike made sure that Java's regular expressions passed virtually every one of the 30K Perl regular expression tests (and ran faster).

The Google common-library Guava contains also a Splitter which is:

• simpler to use
• maintained by Google (and not by you)

So it may worth being checked out. From their initial rough documentation (pdf):

JDK has this:

String[] pieces = "foo.bar".split("\\.");

It's fine to use this if you want exactly what it does: - regular expression - result as an array - its way of handling empty pieces

Mini-puzzler: ",a,,b,".split(",") returns...

(a) "", "a", "", "b", ""
(b) null, "a", null, "b", null
(c) "a", null, "b"
(d) "a", "b"
(e) None of the above

Answer: (e) None of the above.

",a,,b,".split(",")
returns
"", "a", "", "b"

Only trailing empties are skipped! (Who knows the workaround to prevent the skipping? It's a fun one...)

In any case, our Splitter is simply more flexible: The default behavior is simplistic:

Splitter.on(',').split(" foo, ,bar, quux,")
--> [" foo", " ", "bar", " quux", ""]

If you want extra features, ask for them!

Splitter.on(',')
.trimResults()
.omitEmptyStrings()
.split(" foo, ,bar, quux,")
--> ["foo", "bar", "quux"]

Order of config methods doesn't matter -- during splitting, trimming happens before checking for empties.

Answer 5

Another candidate solution using a regex. Retains token order, correctly matches multiple tokens of the same type in a row. The downside is that the regex is kind of nasty.

package javaapplication2;

import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class JavaApplication2 {

    /**
     * @param args the command line arguments
     */
    public static void main(String[] args) {
        String num = "58.5+variable-+98*78/96+a/78.7-3443*12-3";

        // Terrifying regex:
        //  (a)|(b)|(c) match a or b or c
        // where
        //   (a) is one or more digits optionally followed by a decimal point
        //       followed by one or more digits: (\d+(\.\d+)?)
        //   (b) is one of the set + * / - occurring once: ([+*/-])
        //   (c) is a sequence of one or more lowercase latin letter: ([a-z]+)
        Pattern tokenPattern = Pattern.compile("(\\d+(\\.\\d+)?)|([+*/-])|([a-z]+)");
        Matcher tokenMatcher = tokenPattern.matcher(num);

        List<String> tokens = new ArrayList<>();

        while (!tokenMatcher.hitEnd()) {
            if (tokenMatcher.find()) {
                tokens.add(tokenMatcher.group());
            } else {
                // report error
                break;
            }
        }

        System.out.println(tokens);
    }
}

Sample output:

[58.5, +, variable, -, +, 98, *, 78, /, 96, +, a, /, 78.7, -, 3443, *, 12, -, 3]

Answer 6

    String expression = "((A+B)*C-D)*E";
    expression = expression.replaceAll("\\+", "~+~");
    expression = expression.replaceAll("\\*", "~*~");
    expression = expression.replaceAll("-", "~-~");
    expression = expression.replaceAll("/+", "~/~");
    expression = expression.replaceAll("\\(", "~(~"); //also you can use [(] instead of \\(
    expression = expression.replaceAll("\\)", "~)~"); //also you can use [)] instead of \\)
    expression = expression.replaceAll("~~", "~");
    if(expression.startsWith("~")) {
        expression = expression.substring(1);
    }

    String[] expressionArray = expression.split("~");
    System.out.println(Arrays.toString(expressionArray));