From 68fd4965162d87dd866de53113c05189ca00820c Mon Sep 17 00:00:00 2001
From: Jeremy Fincher <jemfinch@users.sourceforge.net>
Date: Mon, 20 Oct 2003 04:17:14 +0000
Subject: [PATCH] Added memoization optimization to tokenize function.

---
 src/callbacks.py | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/src/callbacks.py b/src/callbacks.py
index 90f912544..d29b7a413 100644
--- a/src/callbacks.py
+++ b/src/callbacks.py
@@ -284,19 +284,19 @@ def tokenize(s):
     debug.msg('tokenize took %s seconds.' % (time.time() - start), 'verbose')
     return _lastTokenizeResult
 
-def tokenize(s):
-    """A utility function to create a Tokenizer and tokenize a string."""
-    start = time.time()
-    try:
-        if conf.enablePipeSyntax:
-            tokens = '|'
-        else:
-            tokens = ''
-        args = Tokenizer(tokens).tokenize(s)
-    except ValueError, e:
-        raise SyntaxError, str(e)
-    #debug.msg('tokenize took %s seconds.' % (time.time() - start), 'verbose')
-    return args
+## def tokenize(s):
+##     """A utility function to create a Tokenizer and tokenize a string."""
+##     start = time.time()
+##     try:
+##         if conf.enablePipeSyntax:
+##             tokens = '|'
+##         else:
+##             tokens = ''
+##         args = Tokenizer(tokens).tokenize(s)
+##     except ValueError, e:
+##         raise SyntaxError, str(e)
+##     #debug.msg('tokenize took %s seconds.' % (time.time() - start), 'verbose')
+##     return args
 
 def getCommands(tokens):
     """Given tokens as output by tokenize, returns the command names."""