Simple binding to Python's internal tokenizer (work in progress).

To build this under Windows, you need to expose the PyTokenizer API,
by patching the Parser/tokenizer.h file before building the DLL.  See
the patch below.

To build this on any other platform, add the Parser library to the
include path, or copy the "tokenizer.h" file to the current directory.

Index: tokenizer.h
===================================================================
--- tokenizer.h (revision 514)
+++ tokenizer.h (working copy)
@@ -54,10 +54,10 @@
        const char* str;
 };

-extern struct tok_state *PyTokenizer_FromString(const char *);
-extern struct tok_state *PyTokenizer_FromFile(FILE *, char *, char *);
-extern void PyTokenizer_Free(struct tok_state *);
-extern int PyTokenizer_Get(struct tok_state *, char **, char **);
+PyAPI_FUNC(struct tok_state *) PyTokenizer_FromString(const char *);
+PyAPI_FUNC(struct tok_state *) PyTokenizer_FromFile(FILE *, char *, char *);
+PyAPI_FUNC(void) PyTokenizer_Free(struct tok_state *);
+PyAPI_FUNC(int) PyTokenizer_Get(struct tok_state *, char **, char **);

 #ifdef __cplusplus
 }