]> git.donarmstrong.com Git - roundcube.git/blobdiff - plugins/managesieve/lib/rcube_sieve.php
Imported Upstream version 0.5.2+dfsg
[roundcube.git] / plugins / managesieve / lib / rcube_sieve.php
index 89ae7d496e9ea2d4378383218a74cbeb06b2dd80..64bdb20f02384028655dd7c3bc0ff10f913ba751 100644 (file)
@@ -1,11 +1,11 @@
 <?php
 
-/*
+/**
   Classes for managesieve operations (using PEAR::Net_Sieve)
 
   Author: Aleksander Machniak <alec@alec.pl>
 
-  $Id: rcube_sieve.php 4241 2010-11-20 17:59:50Z alec $
+  $Id: rcube_sieve.php 4555 2011-02-16 10:48:11Z alec $
 
 */
 
@@ -57,7 +57,7 @@ class rcube_sieve
             $this->sieve->setDebug(true, array($this, 'debug_handler'));
         }
 
-        if (PEAR::isError($this->sieve->connect($host, $port, NULL, $usetls))) {
+        if (PEAR::isError($this->sieve->connect($host, $port, null, $usetls))) {
             return $this->_set_error(SIEVE_ERROR_CONNECTION);
         }
 
@@ -414,12 +414,17 @@ class rcube_sieve_script
      * @param  string  Script's text content
      * @param  array   Disabled extensions
      */
-    public function __construct($script, $disabled=NULL)
+    public function __construct($script, $disabled=null)
     {
-        if (!empty($disabled))
-            foreach ($disabled as $ext)
-                if (($idx = array_search($ext, $this->supported)) !== false)
+        if (!empty($disabled)) {
+            // we're working on lower-cased names
+            $disabled = array_map('strtolower', (array) $disabled);
+            foreach ($disabled as $ext) {
+                if (($idx = array_search($ext, $this->supported)) !== false) {
                     unset($this->supported[$idx]);
+                }
+            }
+        }
 
         $this->content = $this->_parse_text($script);
     }
@@ -513,14 +518,11 @@ class rcube_sieve_script
                     $tests[$i] .= 'size :' . ($test['type']=='under' ? 'under ' : 'over ') . $test['arg'];
                     break;
                 case 'true':
-                    $tests[$i] .= ($test['not'] ? 'not true' : 'true');
+                    $tests[$i] .= ($test['not'] ? 'false' : 'true');
                     break;
                 case 'exists':
                     $tests[$i] .= ($test['not'] ? 'not ' : '');
-                    if (is_array($test['arg']))
-                        $tests[$i] .= 'exists ["' . implode('", "', $this->_escape_string($test['arg'])) . '"]';
-                    else
-                        $tests[$i] .= 'exists "' . $this->_escape_string($test['arg']) . '"';
+                    $tests[$i] .= 'exists ' . self::escape_string($test['arg']);
                     break;
                 case 'header':
                     $tests[$i] .= ($test['not'] ? 'not ' : '');
@@ -533,33 +535,34 @@ class rcube_sieve_script
                     }
                     else
                         $tests[$i] .= 'header :' . $test['type'];
-                    
-                    if (is_array($test['arg1']))
-                        $tests[$i] .= ' ["' . implode('", "', $this->_escape_string($test['arg1'])) . '"]';
-                    else
-                        $tests[$i] .= ' "' . $this->_escape_string($test['arg1']) . '"';
-
-                    if (is_array($test['arg2']))
-                        $tests[$i] .= ' ["' . implode('", "', $this->_escape_string($test['arg2'])) . '"]';
-                    else
-                        $tests[$i] .= ' "' . $this->_escape_string($test['arg2']) . '"';
 
+                    $tests[$i] .= ' ' . self::escape_string($test['arg1']);
+                    $tests[$i] .= ' ' . self::escape_string($test['arg2']);
                     break;
                 }
                 $i++;
             }
 
-//          $script .= ($idx>0 ? 'els' : '').($rule['join'] ? 'if allof (' : 'if anyof (');
             // disabled rule: if false #....
-            $script .= 'if' . ($rule['disabled'] ? ' false #' : '');
-            $script .= $rule['join'] ? ' allof (' : ' anyof (';
-            if (sizeof($tests) > 1)
-                $script .= implode(", ", $tests);
-            else if (sizeof($tests))
-                $script .= $tests[0];
-            else
-                $script .= 'true';
-            $script .= ")\n{\n";
+            $script .= 'if ' . ($rule['disabled'] ? 'false # ' : '');
+
+            if (empty($tests)) {
+                $tests_str = 'true';
+            }
+            else if (count($tests) > 1) {
+                $tests_str = implode(', ', $tests);
+            }
+            else {
+                $tests_str = $tests[0];
+            }
+
+            if ($rule['join'] || count($tests) > 1) {
+                $script .= sprintf('%s (%s)', $rule['join'] ? 'allof' : 'anyof', $tests_str);
+            }
+            else {
+                $script .= $tests_str;
+            }
+            $script .= "\n{\n";
 
             // action(s)
             foreach ($rule['actions'] as $action) {
@@ -571,7 +574,7 @@ class rcube_sieve_script
                         $script .= ':copy ';
                         array_push($exts, 'copy');
                     }
-                    $script .= "\"" . $this->_escape_string($action['target']) . "\";\n";
+                    $script .= self::escape_string($action['target']) . ";\n";
                     break;
                 case 'redirect':
                     $script .= "\tredirect ";
@@ -579,15 +582,13 @@ class rcube_sieve_script
                         $script .= ':copy ';
                         array_push($exts, 'copy');
                     }
-                    $script .= "\"" . $this->_escape_string($action['target']) . "\";\n";
+                    $script .= self::escape_string($action['target']) . ";\n";
                     break;
                 case 'reject':
                 case 'ereject':
                     array_push($exts, $action['type']);
-                    if (strpos($action['target'], "\n")!==false)
-                        $script .= "\t".$action['type']." text:\n" . $action['target'] . "\n.\n;\n";
-                    else
-                        $script .= "\t".$action['type']." \"" . $this->_escape_string($action['target']) . "\";\n";
+                    $script .= "\t".$action['type']." "
+                        . self::escape_string($action['target']) . ";\n";
                     break;
                 case 'keep':
                 case 'discard':
@@ -597,22 +598,19 @@ class rcube_sieve_script
                 case 'vacation':
                     array_push($exts, 'vacation');
                     $script .= "\tvacation";
-                    if ($action['days'])
+                    if (!empty($action['days']))
                         $script .= " :days " . $action['days'];
-                    if ($action['addresses'])
-                        $script .= " :addresses " . $this->_print_list($action['addresses']);
-                    if ($action['subject'])
-                        $script .= " :subject \"" . $this->_escape_string($action['subject']) . "\"";
-                    if ($action['handle'])
-                        $script .= " :handle \"" . $this->_escape_string($action['handle']) . "\"";
-                    if ($action['from'])
-                        $script .= " :from \"" . $this->_escape_string($action['from']) . "\"";
-                    if ($action['mime'])
+                    if (!empty($action['addresses']))
+                        $script .= " :addresses " . self::escape_string($action['addresses']);
+                    if (!empty($action['subject']))
+                        $script .= " :subject " . self::escape_string($action['subject']);
+                    if (!empty($action['handle']))
+                        $script .= " :handle " . self::escape_string($action['handle']);
+                    if (!empty($action['from']))
+                        $script .= " :from " . self::escape_string($action['from']);
+                    if (!empty($action['mime']))
                         $script .= " :mime";
-                    if (strpos($action['reason'], "\n")!==false)
-                        $script .= " text:\n" . $action['reason'] . "\n.\n;\n";
-                    else
-                        $script .= " \"" . $this->_escape_string($action['reason']) . "\";\n";
+                    $script .= " " . self::escape_string($action['reason']) . ";\n";
                     break;
                 }
             }
@@ -656,9 +654,6 @@ class rcube_sieve_script
         $i = 0;
         $content = array();
 
-        // remove C comments
-        $script = preg_replace('|/\*.*?\*/|sm', '', $script);
-
         // tokenize rules
         if ($tokens = preg_split('/(# rule:\[.*\])\r?\n/', $script, -1, PREG_SPLIT_DELIM_CAPTURE)) {
             foreach($tokens as $token) {
@@ -686,31 +681,118 @@ class rcube_sieve_script
      */
     private function _tokenize_rule($content)
     {
-        $result = NULL;
+        $cond = strtolower(self::tokenize($content, 1));
+
+        if ($cond != 'if' && $cond != 'elsif' && $cond != 'else') {
+            return null;
+        }
+
+        $disabled = false;
+        $join     = false;
+
+        // disabled rule (false + comment): if false # .....
+        if (preg_match('/^\s*false\s+#/i', $content)) {
+            $content = preg_replace('/^\s*false\s+#\s*/i', '', $content);
+            $disabled = true;
+        }
+
+        while (strlen($content)) {
+            $tokens = self::tokenize($content, true);
+            $separator = array_pop($tokens);
+
+            if (!empty($tokens)) {
+                $token = array_shift($tokens);
+            }
+            else {
+                $token = $separator;
+            }
+
+            $token = strtolower($token);
+
+            if ($token == 'not') {
+                $not = true;
+                $token = strtolower(array_shift($tokens));
+            }
+            else {
+                $not = false;
+            }
+
+            switch ($token) {
+            case 'allof':
+                $join = true;
+                break;
+            case 'anyof':
+                break;
+
+            case 'size':
+                $size = array('test' => 'size', 'not'  => $not);
+                for ($i=0, $len=count($tokens); $i<$len; $i++) {
+                    if (!is_array($tokens[$i])
+                        && preg_match('/^:(under|over)$/i', $tokens[$i])
+                    ) {
+                        $size['type'] = strtolower(substr($tokens[$i], 1));
+                    }
+                    else {
+                        $size['arg'] = $tokens[$i];
+                    }
+                }
+
+                $tests[] = $size;
+                break;
+
+            case 'header':
+                $header = array('test' => 'header', 'not' => $not, 'arg1' => '', 'arg2' => '');
+                for ($i=0, $len=count($tokens); $i<$len; $i++) {
+                    if (!is_array($tokens[$i]) && preg_match('/^:comparator$/i', $tokens[$i])) {
+                        $i++;
+                    }
+                    else if (!is_array($tokens[$i]) && preg_match('/^:(count|value)$/i', $tokens[$i])) {
+                        $header['type'] = strtolower(substr($tokens[$i], 1)) . '-' . $tokens[++$i];
+                    }
+                    else if (!is_array($tokens[$i]) && preg_match('/^:(is|contains|matches)$/i', $tokens[$i])) {
+                        $header['type'] = strtolower(substr($tokens[$i], 1));
+                    }
+                    else {
+                        $header['arg1'] = $header['arg2'];
+                        $header['arg2'] = $tokens[$i];
+                    }
+                }
 
-        if (preg_match('/^(if|elsif|else)\s+((true|false|not\s+true|allof|anyof|exists|header|not|size)(.*))\s+\{(.*)\}$/sm',
-            trim($content), $matches)) {
+                $tests[] = $header;
+                break;
 
-            $tests = trim($matches[2]);
+            case 'exists':
+                $tests[] = array('test' => 'exists', 'not'  => $not,
+                    'arg'  => array_pop($tokens));
+                break;
+
+            case 'true':
+                $tests[] = array('test' => 'true', 'not'  => $not);
+                break;
 
-            // disabled rule (false + comment): if false #.....
-            if ($matches[3] == 'false') {
-                $tests = preg_replace('/^false\s+#\s+/', '', $tests);
-                $disabled = true;
+            case 'false':
+                $tests[] = array('test' => 'true', 'not'  => !$not);
+                break;
+            }
+
+            // goto actions...
+            if ($separator == '{') {
+                break;
             }
-            else
-                $disabled = false;
-
-            list($tests, $join) = $this->_parse_tests($tests);
-            $actions = $this->_parse_actions(trim($matches[5]));
-
-            if ($tests && $actions)
-                $result = array(
-                    'type'     => $matches[1],
-                    'tests'    => $tests,
-                    'actions'  => $actions,
-                    'join'     => $join,
-                    'disabled' => $disabled,
+        }
+
+        // ...and actions block
+        if ($tests) {
+            $actions = $this->_parse_actions($content);
+        }
+
+        if ($tests && $actions) {
+            $result = array(
+                'type'     => $cond,
+                'tests'    => $tests,
+                'actions'  => $actions,
+                'join'     => $join,
+                'disabled' => $disabled,
             );
         }
 
@@ -725,94 +807,76 @@ class rcube_sieve_script
      */
     private function _parse_actions($content)
     {
-        $result = NULL;
-
-        // supported actions
-        $patterns[] = '^\s*discard;';
-        $patterns[] = '^\s*keep;';
-        $patterns[] = '^\s*stop;';
-        $patterns[] = '^\s*redirect\s+(.*?[^\\\]);';
-        if (in_array('fileinto', $this->supported))
-            $patterns[] = '^\s*fileinto\s+(.*?[^\\\]);';
-        if (in_array('reject', $this->supported)) {
-            $patterns[] = '^\s*reject\s+text:(.*)\n\.\n;';
-            $patterns[] = '^\s*reject\s+(.*?[^\\\]);';
-            $patterns[] = '^\s*ereject\s+text:(.*)\n\.\n;';
-            $patterns[] = '^\s*ereject\s+(.*?[^\\\]);';
-        }
-        if (in_array('vacation', $this->supported))
-            $patterns[] = '^\s*vacation\s+(.*?[^\\\]);';
+        $result = null;
 
-        $pattern = '/(' . implode('\s*$)|(', $patterns) . '$\s*)/ms';
+        while (strlen($content)) {
+            $tokens = self::tokenize($content, true);
+            $separator = array_pop($tokens);
 
-        // parse actions body
-        if (preg_match_all($pattern, $content, $mm, PREG_SET_ORDER)) {
-            foreach ($mm as $m) {
-                $content = trim($m[0]);
+            if (!empty($tokens)) {
+                $token = array_shift($tokens);
+            }
+            else {
+                $token = $separator;
+            }
 
-                if(preg_match('/^(discard|keep|stop)/', $content, $matches)) {
-                    $result[] = array('type' => $matches[1]);
-                }
-                else if(preg_match('/^fileinto/', $content)) {
-                    $target = $m[sizeof($m)-1];
-                    $copy = false;
-                    if (preg_match('/^:copy\s+/', $target)) {
-                        $target = preg_replace('/^:copy\s+/', '', $target);
+            switch ($token) {
+            case 'discard':
+            case 'keep':
+            case 'stop':
+                $result[] = array('type' => $token);
+                break;
+
+            case 'fileinto':
+            case 'redirect':
+                $copy   = false;
+                $target = '';
+
+                for ($i=0, $len=count($tokens); $i<$len; $i++) {
+                    if (strtolower($tokens[$i]) == ':copy') {
                         $copy = true;
                     }
-                    $result[] = array('type' => 'fileinto', 'copy' => $copy,
-                        'target' => $this->_parse_string($target));
-                }
-                else if(preg_match('/^redirect/', $content)) {
-                    $target = $m[sizeof($m)-1];
-                    $copy = false;
-                    if (preg_match('/^:copy\s+/', $target)) {
-                        $target = preg_replace('/^:copy\s+/', '', $target);
-                        $copy = true;
+                    else {
+                        $target = $tokens[$i];
                     }
-                    $result[] = array('type' => 'redirect', 'copy' => $copy,
-                        'target' => $this->_parse_string($target));
-                }
-                else if(preg_match('/^(reject|ereject)\s+(.*);$/sm', $content, $matches)) {
-                    $result[] = array('type' => $matches[1], 'target' => $this->_parse_string($matches[2]));
                 }
-                else if(preg_match('/^vacation\s+(.*);$/sm', $content, $matches)) {
-                    $vacation = array('type' => 'vacation');
 
-                    if (preg_match('/:days\s+([0-9]+)/', $content, $vm)) {
-                        $vacation['days'] = $vm[1];
-                        $content = preg_replace('/:days\s+([0-9]+)/', '', $content);
+                $result[] = array('type' => $token, 'copy' => $copy,
+                    'target' => $target);
+                break;
+
+            case 'reject':
+            case 'ereject':
+                $result[] = array('type' => $token, 'target' => array_pop($tokens));
+                break;
+
+            case 'vacation':
+                $vacation = array('type' => 'vacation', 'reason' => array_pop($tokens));
+
+                for ($i=0, $len=count($tokens); $i<$len; $i++) {
+                    $tok = strtolower($tokens[$i]);
+                    if ($tok == ':days') {
+                        $vacation['days'] = $tokens[++$i];
                     }
-                    if (preg_match('/:subject\s+"(.*?[^\\\])"/', $content, $vm)) {
-                        $vacation['subject'] = $vm[1];
-                        $content = preg_replace('/:subject\s+"(.*?[^\\\])"/', '', $content);
+                    else if ($tok == ':subject') {
+                        $vacation['subject'] = $tokens[++$i];
                     }
-                    if (preg_match('/:addresses\s+\[(.*?[^\\\])\]/', $content, $vm)) {
-                        $vacation['addresses'] = $this->_parse_list($vm[1]);
-                        $content = preg_replace('/:addresses\s+\[(.*?[^\\\])\]/', '', $content);
+                    else if ($tok == ':addresses') {
+                        $vacation['addresses'] = $tokens[++$i];
                     }
-                    if (preg_match('/:handle\s+"(.*?[^\\\])"/', $content, $vm)) {
-                        $vacation['handle'] = $vm[1];
-                        $content = preg_replace('/:handle\s+"(.*?[^\\\])"/', '', $content);
+                    else if ($tok == ':handle') {
+                        $vacation['handle'] = $tokens[++$i];
                     }
-                    if (preg_match('/:from\s+"(.*?[^\\\])"/', $content, $vm)) {
-                        $vacation['from'] = $vm[1];
-                        $content = preg_replace('/:from\s+"(.*?[^\\\])"/', '', $content);
+                    else if ($tok == ':from') {
+                        $vacation['from'] = $tokens[++$i];
                     }
-
-                    $content = preg_replace('/^vacation/', '', $content);
-                    $content = preg_replace('/;$/', '', $content);
-                    $content = trim($content);
-
-                    if (preg_match('/^:mime/', $content, $vm)) {
+                    else if ($tok == ':mime') {
                         $vacation['mime'] = true;
-                        $content = preg_replace('/^:mime/', '', $content);
                     }
-
-                    $vacation['reason'] = $this->_parse_string($content);
-
-                    $result[] = $vacation;
                 }
+
+                $result[] = $vacation;
+                break;
             }
         }
 
@@ -820,171 +884,196 @@ class rcube_sieve_script
     }
 
     /**
-     * Parse test/conditions section
+     * Escape special chars into quoted string value or multi-line string
+     * or list of strings
      *
-     * @param string Text
+     * @param string $str Text or array (list) of strings
+     *
+     * @return string Result text
      */
-    private function _parse_tests($content)
+    static function escape_string($str)
     {
-        $result = NULL;
+        if (is_array($str) && count($str) > 1) {
+            foreach($str as $idx => $val)
+                $str[$idx] = self::escape_string($val);
 
-        // lists
-        if (preg_match('/^(allof|anyof)\s+\((.*)\)$/sm', $content, $matches)) {
-            $content = $matches[2];
-            $join = $matches[1]=='allof' ? true : false;
+            return '[' . implode(',', $str) . ']';
         }
-        else
-            $join = false;
-
-        // supported tests regular expressions
-        // TODO: comparators, envelope
-        $patterns[] = '(not\s+)?(exists)\s+\[(.*?[^\\\])\]';
-        $patterns[] = '(not\s+)?(exists)\s+(".*?[^\\\]")';
-        $patterns[] = '(not\s+)?(true)';
-        $patterns[] = '(not\s+)?(size)\s+:(under|over)\s+([0-9]+[KGM]{0,1})';
-        $patterns[] = '(not\s+)?(header)\s+:(contains|is|matches)((\s+))\[(.*?[^\\\]")\]\s+\[(.*?[^\\\]")\]';
-        $patterns[] = '(not\s+)?(header)\s+:(contains|is|matches)((\s+))(".*?[^\\\]")\s+(".*?[^\\\]")';
-        $patterns[] = '(not\s+)?(header)\s+:(contains|is|matches)((\s+))\[(.*?[^\\\]")\]\s+(".*?[^\\\]")';
-        $patterns[] = '(not\s+)?(header)\s+:(contains|is|matches)((\s+))(".*?[^\\\]")\s+\[(.*?[^\\\]")\]';
-               $patterns[] = '(not\s+)?(header)\s+:(count\s+"[gtleqn]{2}"|value\s+"[gtleqn]{2}")(\s+:comparator\s+"(.*?[^\\\])")?\s+\[(.*?[^\\\]")\]\s+\[(.*?[^\\\]")\]';
-               $patterns[] = '(not\s+)?(header)\s+:(count\s+"[gtleqn]{2}"|value\s+"[gtleqn]{2}")(\s+:comparator\s+"(.*?[^\\\])")?\s+(".*?[^\\\]")\s+(".*?[^\\\]")';
-               $patterns[] = '(not\s+)?(header)\s+:(count\s+"[gtleqn]{2}"|value\s+"[gtleqn]{2}")(\s+:comparator\s+"(.*?[^\\\])")?\s+\[(.*?[^\\\]")\]\s+(".*?[^\\\]")';
-               $patterns[] = '(not\s+)?(header)\s+:(count\s+"[gtleqn]{2}"|value\s+"[gtleqn]{2}")(\s+:comparator\s+"(.*?[^\\\])")?\s+(".*?[^\\\]")\s+\[(.*?[^\\\]")\]';
-
-        // join patterns...
-        $pattern = '/(' . implode(')|(', $patterns) . ')/';
-
-        // ...and parse tests list
-        if (preg_match_all($pattern, $content, $matches, PREG_SET_ORDER)) {
-            foreach ($matches as $match) {
-                $size = sizeof($match);
-
-                if (preg_match('/^(not\s+)?size/', $match[0])) {
-                    $result[] = array(
-                        'test' => 'size',
-                        'not'  => $match[$size-4] ? true : false,
-                        'type' => $match[$size-2], // under/over
-                        'arg'  => $match[$size-1], // value
-                    );
-                }
-                else if (preg_match('/^(not\s+)?header/', $match[0])) {
-                    $type = $match[$size-5];
-                    if (preg_match('/^(count|value)\s+"([gtleqn]{2})"/', $type, $m))
-                        $type = $m[1] . '-' . $m[2];
-                    
-                    $result[] = array(
-                        'test' => 'header',
-                        'type' => $type, // is/contains/matches
-                                               'not'  => $match[$size-7] ? true : false,
-                        'arg1' => $this->_parse_list($match[$size-2]), // header(s)
-                        'arg2' => $this->_parse_list($match[$size-1]), // string(s)
-                    );
-                }
-                else if (preg_match('/^(not\s+)?exists/', $match[0])) {
-                    $result[] = array(
-                        'test' => 'exists',
-                        'not'  => $match[$size-3] ? true : false,
-                        'arg'  => $this->_parse_list($match[$size-1]), // header(s)
-                    );
-                }
-                else if (preg_match('/^(not\s+)?true/', $match[0])) {
-                    $result[] = array(
-                        'test' => 'true',
-                        'not'  => $match[$size-2] ? true : false,
-                    );
-                }
-            }
+        else if (is_array($str)) {
+            $str = array_pop($str);
         }
 
-        return array($result, $join);
+        // multi-line string
+        if (preg_match('/[\r\n\0]/', $str) || strlen($str) > 1024) {
+            return sprintf("text:\n%s\n.\n", self::escape_multiline_string($str));
+        }
+        // quoted-string
+        else {
+            $replace = array('\\' => '\\\\', '"' => '\\"');
+            $str = str_replace(array_keys($replace), array_values($replace), $str);
+            return '"' . $str . '"';
+        }
     }
 
     /**
-     * Parse string value
+     * Escape special chars in multi-line string value
+     *
+     * @param string $str Text
      *
-     * @param string Text
+     * @return string Text
      */
-    private function _parse_string($content)
+    static function escape_multiline_string($str)
     {
-        $text = '';
-        $content = trim($content);
+        $str = preg_split('/(\r?\n)/', $str, -1, PREG_SPLIT_DELIM_CAPTURE);
 
-        if (preg_match('/^text:(.*)\.$/sm', $content, $matches))
-            $text = trim($matches[1]);
-        else if (preg_match('/^"(.*)"$/', $content, $matches))
-            $text = str_replace('\"', '"', $matches[1]);
+        foreach ($str as $idx => $line) {
+            // dot-stuffing
+            if (isset($line[0]) && $line[0] == '.') {
+                $str[$idx] = '.' . $line;
+            }
+        }
 
-        return $text;
+        return implode($str);
     }
 
     /**
-     * Escape special chars in string value
+     * Splits script into string tokens
+     *
+     * @param string &$str    The script
+     * @param mixed  $num     Number of tokens to return, 0 for all
+     *                        or True for all tokens until separator is found.
+     *                        Separator will be returned as last token.
+     * @param int    $in_list Enable to called recursively inside a list
      *
-     * @param string Text
+     * @return mixed Tokens array or string if $num=1
      */
-    private function _escape_string($content)
+    static function tokenize(&$str, $num=0, $in_list=false)
     {
-        $replace['/"/'] = '\\"';
+        $result = array();
 
-        if (is_array($content)) {
-            for ($x=0, $y=sizeof($content); $x<$y; $x++)
-                $content[$x] = preg_replace(array_keys($replace),
-                    array_values($replace), $content[$x]);
+        // remove spaces from the beginning of the string
+        while (($str = ltrim($str)) !== ''
+            && (!$num || $num === true || count($result) < $num)
+        ) {
+            switch ($str[0]) {
 
-            return $content;
-        }
-        else
-            return preg_replace(array_keys($replace), array_values($replace), $content);
-    }
+            // Quoted string
+            case '"':
+                $len = strlen($str);
 
-    /**
-     * Parse string or list of strings to string or array of strings
-     *
-     * @param string Text
-     */
-    private function _parse_list($content)
-    {
-        $result = array();
+                for ($pos=1; $pos<$len; $pos++) {
+                    if ($str[$pos] == '"') {
+                        break;
+                    }
+                    if ($str[$pos] == "\\") {
+                        if ($str[$pos + 1] == '"' || $str[$pos + 1] == "\\") {
+                            $pos++;
+                        }
+                    }
+                }
+                if ($str[$pos] != '"') {
+                    // error
+                }
+                // we need to strip slashes for a quoted string
+                $result[] = stripslashes(substr($str, 1, $pos - 1));
+                $str      = substr($str, $pos + 1);
+                break;
 
-        for ($x=0, $len=strlen($content); $x<$len; $x++) {
-            switch ($content[$x]) {
-            case '\\':
-                $str .= $content[++$x];
+            // Parenthesized list
+            case '[':
+                $str = substr($str, 1);
+                $result[] = self::tokenize($str, 0, true);
                 break;
-            case '"':
-                if (isset($str)) {
-                    $result[] = $str;
-                    unset($str);
+            case ']':
+                $str = substr($str, 1);
+                return $result;
+                break;
+
+            // list/test separator
+            case ',':
+            // command separator
+            case ';':
+            // block/tests-list
+            case '(':
+            case ')':
+            case '{':
+            case '}':
+                $sep = $str[0];
+                $str = substr($str, 1);
+                if ($num === true) {
+                    $result[] = $sep;
+                    break 2; 
+                }
+                break;
+
+            // bracket-comment
+            case '/':
+                if ($str[1] == '*') {
+                    if ($end_pos = strpos($str, '*/')) {
+                        $str = substr($str, $end_pos + 2);
+                    }
+                    else {
+                        // error
+                        $str = '';
+                    }
                 }
-                else
-                    $str = '';
                 break;
+
+            // hash-comment
+            case '#':
+                if ($lf_pos = strpos($str, "\n")) {
+                    $str = substr($str, $lf_pos);
+                    break;
+                }
+                else {
+                    $str = '';
+                }
+
+            // String atom
             default:
-                if(isset($str))
-                    $str .= $content[$x];
-            break;
+                // empty or one character
+                if ($str === '') {
+                    break 2;
+                }
+                if (strlen($str) < 2) {
+                    $result[] = $str;
+                    $str = '';
+                    break;
+                }
+
+                // tag/identifier/number
+                if (preg_match('/^([a-z0-9:_]+)/i', $str, $m)) {
+                    $str = substr($str, strlen($m[1]));
+
+                    if ($m[1] != 'text:') {
+                        $result[] = $m[1];
+                    }
+                    // multiline string
+                    else {
+                        // possible hash-comment after "text:"
+                        if (preg_match('/^( |\t)*(#[^\n]+)?\n/', $str, $m)) {
+                            $str = substr($str, strlen($m[0]));
+                        }
+                        // get text until alone dot in a line
+                        if (preg_match('/^(.*)\r?\n\.\r?\n/sU', $str, $m)) {
+                            $text = $m[1];
+                            // remove dot-stuffing
+                            $text = str_replace("\n..", "\n.", $text);
+                            $str = substr($str, strlen($m[0]));
+                        }
+                        else {
+                            $text = '';
+                        }
+
+                        $result[] = $text;
+                    }
+                }
+
+                break;
             }
         }
 
-        if (sizeof($result)>1)
-            return $result;
-        else if (sizeof($result) == 1)
-            return $result[0];
-        else
-            return NULL;
+        return $num === 1 ? (isset($result[0]) ? $result[0] : null) : $result;
     }
 
-    /**
-     * Convert array of elements to list of strings
-     *
-     * @param string Text
-     */
-    private function _print_list($list)
-    {
-        $list = (array) $list;
-        foreach($list as $idx => $val)
-            $list[$idx] = $this->_escape_string($val);
-
-        return '["' . implode('","', $list) . '"]';
-    }
 }