Configure: teach the tokenizer to handle other separators than spaces
@@ -3508,32 +3508,43 @@ sub collect_information {
}
# tokenize($line)
+# tokenize($line,$separator)
# $line is a line of text to split up into tokens
-# returns a list of tokens
+# $separator [optional] is a regular expression that separates the tokens,
+# the default being spaces. Do not use quotes of any kind as separators,
+# that will give undefined results.
+# Returns a list of tokens.
#
-# Tokens are divided by spaces. If the tokens include spaces, they
-# have to be quoted with single or double quotes. Double quotes
-# inside a double quoted token must be escaped. Escaping is done
+# Tokens are divided by separator (spaces by default). If the tokens include
+# the separators, they have to be quoted with single or double quotes.
+# Double quotes inside a double quoted token must be escaped. Escaping is done
# with backslash.
# Basically, the same quoting rules apply for " and ' as in any
# Unix shell.
sub tokenize {
my $line = my $debug_line = shift;
+ my $separator = shift // qr|\s+|;
my @result = ();
- while ($line =~ s|^\s+||, $line ne "") {
+ if ($ENV{CONFIGURE_DEBUG_TOKENIZE}) {
+ print STDERR "DEBUG[tokenize]: \$separator = $separator\n";
+ }
+
+ while ($line =~ s|^${separator}||, $line ne "") {
my $token = "";
- while ($line ne "" && $line !~ m|^\s|) {
+ again:
+ $line =~ m/^(.*?)(${separator}|"|'|$)/;
+ $token .= $1;
+ $line = $2.$';
+
if ($line =~ m/^"((?:[^"\\]+|\\.)*)"/) {
$token .= $1;
$line = $';
+ goto again;
} elsif ($line =~ m/^'([^']*)'/) {
$token .= $1;
$line = $';
- } elsif ($line =~ m/^(\S+)/) {
- $token .= $1;
- $line = $';
- }
+ goto again;
}
push @result, $token;
}
