changeset 106783:fc2b73c71e0f

Add Semantic parsers for Python. * cedet/semantic.el (semantic-new-buffer-setup-functions): Add python parser. * cedet/semantic/wisent/python-wy.el, cedet/semantic/wisent/python.el: New files.

--- a/lisp/ChangeLog	Sat Jan 09 18:53:06 2010 -0500
+++ b/lisp/ChangeLog	Sat Jan 09 19:38:07 2010 -0500
@@ -1,3 +1,13 @@
[phone]  Chong Yidong  <[e-mail]>
+
+	* cedet/semantic.el (semantic-new-buffer-setup-functions): Add
+	python parser.
+
[phone]  Richard Kim  <[e-mail]>
+
+	* cedet/semantic/wisent/python-wy.el:
+	* cedet/semantic/wisent/python.el: New files.
+
 2010-01-09  Chong Yidong  <[e-mail]>
 
 	* man.el (Man-goto-section): Signal error if the section is not

--- a/lisp/cedet/semantic.el	Sat Jan 09 18:53:06 2010 -0500
+++ b/lisp/cedet/semantic.el	Sat Jan 09 19:38:07 2010 -0500
@@ -266,12 +266,13 @@
   :type 'hook)
 
 (defcustom semantic-new-buffer-setup-functions
-  '((js-mode . wisent-javascript-setup-parser)
-    (java-mode . wisent-java-default-setup)
-    (scheme-mode . semantic-default-scheme-setup)
-    (c-mode . semantic-default-c-setup)
+  '((c-mode . semantic-default-c-setup)
     (c++-mode . semantic-default-c-setup)
     (html-mode . semantic-default-html-setup)
+    (java-mode . wisent-java-default-setup)
+    (js-mode . wisent-javascript-setup-parser)
+    (python-mode . wisent-python-default-setup)
+    (scheme-mode . semantic-default-scheme-setup)
     (srecode-template-mode . srecode-template-setup-parser)
     (makefile-automake-mode . semantic-default-make-setup)
     (makefile-gmake-mode . semantic-default-make-setup)

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/lisp/cedet/semantic/wisent/python-wy.el	Sat Jan 09 19:38:07 2010 -0500
@@ -0,0 +1,749 @@
+;;; semantic/wisent/python-wy.el --- Generated parser support file
+
+;; Copyright (C) 2002, 2003, 2004, 2007, 2010
+;; Free Software Foundation, Inc.
+
+;; This file is part of GNU Emacs.
+
+;; GNU Emacs is free software: you can redistribute it and/or modify
+;; it under the terms of the GNU General Public License as published by
+;; the Free Software Foundation, either version 3 of the License, or
+;; (at your option) any later version.
+
+;; GNU Emacs is distributed in the hope that it will be useful,
+;; but WITHOUT ANY WARRANTY; without even the implied warranty of
+;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+;; GNU General Public License for more details.
+
+;; You should have received a copy of the GNU General Public License
+;; along with GNU Emacs.  If not, see <http://www.gnu.org/licenses/>.
+
+;;; Commentary:
+;;
+;; This file was generated from the grammar file
+;; semantic/wisent/wisent-python.wy in the CEDET repository.
+
+;;; Code:
+
+(require 'semantic/lex)
+
+;;; Prologue
+;;
+
+;;; Declarations
+;;
+(defconst wisent-python-wy--keyword-table
+  (semantic-lex-make-keyword-table
+   '(("and" . AND)
+     ("assert" . ASSERT)
+     ("break" . BREAK)
+     ("class" . CLASS)
+     ("continue" . CONTINUE)
+     ("def" . DEF)
+     ("del" . DEL)
+     ("elif" . ELIF)
+     ("else" . ELSE)
+     ("except" . EXCEPT)
+     ("exec" . EXEC)
+     ("finally" . FINALLY)
+     ("for" . FOR)
+     ("from" . FROM)
+     ("global" . GLOBAL)
+     ("if" . IF)
+     ("import" . IMPORT)
+     ("in" . IN)
+     ("is" . IS)
+     ("lambda" . LAMBDA)
+     ("not" . NOT)
+     ("or" . OR)
+     ("pass" . PASS)
+     ("print" . PRINT)
+     ("raise" . RAISE)
+     ("return" . RETURN)
+     ("try" . TRY)
+     ("while" . WHILE)
+     ("yield" . YIELD))
+   '(("yield" summary "Create a generator function")
+     ("while" summary "Start a 'while' loop")
+     ("try" summary "Start of statements protected by exception handlers")
+     ("return" summary "Return from a function")
+     ("raise" summary "Raise an exception")
+     ("print" summary "Print each argument to standard output")
+     ("pass" summary "Statement that does nothing")
+     ("or" summary "Binary logical 'or' operator")
+     ("not" summary "Unary boolean negation operator")
+     ("is" summary "Binary operator that tests for object equality")
+     ("in" summary "Part of 'for' statement ")
+     ("import" summary "Load specified modules")
+     ("if" summary "Start 'if' conditional statement")
+     ("global" summary "Declare one or more symbols as global symbols")
+     ("from" summary "Modify behavior of 'import' statement")
+     ("for" summary "Start a 'for' loop")
+     ("finally" summary "Specify code to be executed after 'try' statements whether or not an exception occured")
+     ("exec" summary "Dynamically execute python code")
+     ("except" summary "Specify exception handlers along with 'try' keyword")
+     ("else" summary "Start the 'else' clause following an 'if' statement")
+     ("elif" summary "Shorthand for 'else if' following an 'if' statement")
+     ("del" summary "Delete specified objects, i.e., undo what assignment did")
+     ("def" summary "Define a new function")
+     ("continue" summary "Skip to the next interation of enclosing for or whilte loop")
+     ("class" summary "Define a new class")
+     ("break" summary "Terminate 'for' or 'while loop")
+     ("assert" summary "Raise AssertionError exception if <expr> is false")
+     ("and" summary "Logical AND binary operator ... ")))
+  "Table of language keywords.")
+
+(defconst wisent-python-wy--token-table
+  (semantic-lex-make-type-table
+   '(("symbol"
+      (NAME))
+     ("number"
+      (NUMBER_LITERAL))
+     ("string"
+      (STRING_LITERAL))
+     ("punctuation"
+      (BACKQUOTE . "`")
+      (ASSIGN . "=")
+      (COMMA . ",")
+      (SEMICOLON . ";")
+      (COLON . ":")
+      (BAR . "|")
+      (TILDE . "~")
+      (PERIOD . ".")
+      (MINUS . "-")
+      (PLUS . "+")
+      (MOD . "%")
+      (DIV . "/")
+      (MULT . "*")
+      (AMP . "&")
+      (GT . ">")
+      (LT . "<")
+      (HAT . "^")
+      (NE . "!=")
+      (LTGT . "<>")
+      (HATEQ . "^=")
+      (OREQ . "|=")
+      (AMPEQ . "&=")
+      (MODEQ . "%=")
+      (DIVEQ . "/=")
+      (MULTEQ . "*=")
+      (MINUSEQ . "-=")
+      (PLUSEQ . "+=")
+      (LE . "<=")
+      (GE . ">=")
+      (EQ . "==")
+      (EXPONENT . "**")
+      (GTGT . ">>")
+      (LTLT . "<<")
+      (DIVDIV . "//")
+      (DIVDIVEQ . "//=")
+      (EXPEQ . "**=")
+      (GTGTEQ . ">>=")
+      (LTLTEQ . "<<="))
+     ("close-paren"
+      (RBRACK . "]")
+      (RBRACE . "}")
+      (RPAREN . ")"))
+     ("open-paren"
+      (LBRACK . "[")
+      (LBRACE . "{")
+      (LPAREN . "("))
+     ("block"
+      (BRACK_BLOCK . "(LBRACK RBRACK)")
+      (BRACE_BLOCK . "(LBRACE RBRACE)")
+      (PAREN_BLOCK . "(LPAREN RPAREN)"))
+     ("indentation"
+      (INDENT_BLOCK . "(INDENT DEDENT)")
+      (DEDENT . "[^:INDENT:]")
+      (INDENT . "^\\s-+"))
+     ("newline"
+      (NEWLINE . "\n"))
+     ("charquote"
+      (BACKSLASH . "\\")))
+   '(("keyword" :declared t)
+     ("symbol" :declared t)
+     ("number" :declared t)
+     ("punctuation" :declared t)
+     ("block" :declared t)))
+  "Table of lexical tokens.")
+
+(defconst wisent-python-wy--parse-table
+  (progn
+    (eval-when-compile
+      (require 'semantic/wisent/comp))
+    (wisent-compile-grammar
+     '((BACKSLASH NEWLINE INDENT DEDENT INDENT_BLOCK PAREN_BLOCK BRACE_BLOCK BRACK_BLOCK LPAREN RPAREN LBRACE RBRACE LBRACK RBRACK LTLTEQ GTGTEQ EXPEQ DIVDIVEQ DIVDIV LTLT GTGT EXPONENT EQ GE LE PLUSEQ MINUSEQ MULTEQ DIVEQ MODEQ AMPEQ OREQ HATEQ LTGT NE HAT LT GT AMP MULT DIV MOD PLUS MINUS PERIOD TILDE BAR COLON SEMICOLON COMMA ASSIGN BACKQUOTE STRING_LITERAL NUMBER_LITERAL NAME AND ASSERT BREAK CLASS CONTINUE DEF DEL ELIF ELSE EXCEPT EXEC FINALLY FOR FROM GLOBAL IF IMPORT IN IS LAMBDA NOT OR PASS PRINT RAISE RETURN TRY WHILE YIELD)
+       nil
+       (goal
+	((NEWLINE))
+	((simple_stmt))
+	((compound_stmt)))
+       (simple_stmt
+	((small_stmt_list semicolon_opt NEWLINE)))
+       (small_stmt_list
+	((small_stmt))
+	((small_stmt_list SEMICOLON small_stmt)))
+       (small_stmt
+	((expr_stmt))
+	((print_stmt))
+	((del_stmt))
+	((pass_stmt))
+	((flow_stmt))
+	((import_stmt))
+	((global_stmt))
+	((exec_stmt))
+	((assert_stmt)))
+       (print_stmt
+	((PRINT print_stmt_trailer)
+	 (wisent-raw-tag
+	  (semantic-tag-new-code $1 nil))))
+       (print_stmt_trailer
+	((test_list_opt)
+	 nil)
+	((GTGT test trailing_test_list_with_opt_comma_opt)
+	 nil))
+       (trailing_test_list_with_opt_comma_opt
+	(nil)
+	((trailing_test_list comma_opt)
+	 nil))
+       (trailing_test_list
+	((COMMA test)
+	 nil)
+	((trailing_test_list COMMA test)
+	 nil))
+       (expr_stmt
+	((testlist expr_stmt_trailer)
+	 (if
+	     (and $2
+		  (stringp $1)
+		  (string-match "^\\(\\sw\\|\\s_\\)+$" $1))
+	     (wisent-raw-tag
+	      (semantic-tag-new-variable $1 nil nil))
+	   (wisent-raw-tag
+	    (semantic-tag-new-code $1 nil)))))
+       (expr_stmt_trailer
+	((augassign testlist))
+	((eq_testlist_zom)))
+       (eq_testlist_zom
+	(nil)
+	((eq_testlist_zom ASSIGN testlist)
+	 (identity $3)))
+       (augassign
+	((PLUSEQ))
+	((MINUSEQ))
+	((MULTEQ))
+	((DIVEQ))
+	((MODEQ))
+	((AMPEQ))
+	((OREQ))
+	((HATEQ))
+	((LTLTEQ))
+	((GTGTEQ))
+	((EXPEQ))
+	((DIVDIVEQ)))
+       (del_stmt
+	((DEL exprlist)
+	 (wisent-raw-tag
+	  (semantic-tag-new-code $1 nil))))
+       (exprlist
+	((expr_list comma_opt)
+	 nil))
+       (expr_list
+	((expr)
+	 nil)
+	((expr_list COMMA expr)
+	 nil))
+       (pass_stmt
+	((PASS)
+	 (wisent-raw-tag
+	  (semantic-tag-new-code $1 nil))))
+       (flow_stmt
+	((break_stmt))
+	((continue_stmt))
+	((return_stmt))
+	((raise_stmt))
+	((yield_stmt)))
+       (break_stmt
+	((BREAK)
+	 (wisent-raw-tag
+	  (semantic-tag-new-code $1 nil))))
+       (continue_stmt
+	((CONTINUE)
+	 (wisent-raw-tag
+	  (semantic-tag-new-code $1 nil))))
+       (return_stmt
+	((RETURN testlist_opt)
+	 (wisent-raw-tag
+	  (semantic-tag-new-code $1 nil))))
+       (testlist_opt
+	(nil)
+	((testlist)
+	 nil))
+       (yield_stmt
+	((YIELD testlist)
+	 (wisent-raw-tag
+	  (semantic-tag-new-code $1 nil))))
+       (raise_stmt
+	((RAISE zero_one_two_or_three_tests)
+	 (wisent-raw-tag
+	  (semantic-tag-new-code $1 nil))))
+       (zero_one_two_or_three_tests
+	(nil)
+	((test zero_one_or_two_tests)
+	 nil))
+       (zero_one_or_two_tests
+	(nil)
+	((COMMA test zero_or_one_comma_test)
+	 nil))
+       (zero_or_one_comma_test
+	(nil)
+	((COMMA test)
+	 nil))
+       (import_stmt
+	((IMPORT dotted_as_name_list)
+	 (wisent-raw-tag
+	  (semantic-tag-new-include $2 nil)))
+	((FROM dotted_name IMPORT star_or_import_as_name_list)
+	 (wisent-raw-tag
+	  (semantic-tag-new-include $2 nil))))
+       (dotted_as_name_list
+	((dotted_as_name))
+	((dotted_as_name_list COMMA dotted_as_name)))
+       (star_or_import_as_name_list
+	((MULT)
+	 nil)
+	((import_as_name_list)
+	 nil))
+       (import_as_name_list
+	((import_as_name)
+	 nil)
+	((import_as_name_list COMMA import_as_name)
+	 nil))
+       (import_as_name
+	((NAME name_name_opt)
+	 nil))
+       (dotted_as_name
+	((dotted_name name_name_opt)))
+       (name_name_opt
+	(nil)
+	((NAME NAME)
+	 nil))
+       (dotted_name
+	((NAME))
+	((dotted_name PERIOD NAME)
+	 (format "%s.%s" $1 $3)))
+       (global_stmt
+	((GLOBAL comma_sep_name_list)
+	 (wisent-raw-tag
+	  (semantic-tag-new-code $1 nil))))
+       (comma_sep_name_list
+	((NAME))
+	((comma_sep_name_list COMMA NAME)))
+       (exec_stmt
+	((EXEC expr exec_trailer)
+	 (wisent-raw-tag
+	  (semantic-tag-new-code $1 nil))))
+       (exec_trailer
+	(nil)
+	((IN test comma_test_opt)
+	 nil))
+       (comma_test_opt
+	(nil)
+	((COMMA test)
+	 nil))
+       (assert_stmt
+	((ASSERT test comma_test_opt)
+	 (wisent-raw-tag
+	  (semantic-tag-new-code $1 nil))))
+       (compound_stmt
+	((if_stmt))
+	((while_stmt))
+	((for_stmt))
+	((try_stmt))
+	((funcdef))
+	((class_declaration)))
+       (if_stmt
+	((IF test COLON suite elif_suite_pair_list else_suite_pair_opt)
+	 (wisent-raw-tag
+	  (semantic-tag-new-code $1 nil))))
+       (elif_suite_pair_list
+	(nil)
+	((elif_suite_pair_list ELIF test COLON suite)
+	 nil))
+       (else_suite_pair_opt
+	(nil)
+	((ELSE COLON suite)
+	 nil))
+       (suite
+	((simple_stmt)
+	 (list $1))
+	((NEWLINE indented_block)
+	 (progn $2)))
+       (indented_block
+	((INDENT_BLOCK)
+	 (semantic-parse-region
+	  (car $region1)
+	  (cdr $region1)
+	  'indented_block_body 1)))
+       (indented_block_body
+	((INDENT)
+	 nil)
+	((DEDENT)
+	 nil)
+	((simple_stmt))
+	((compound_stmt)))
+       (while_stmt
+	((WHILE test COLON suite else_suite_pair_opt)
+	 (wisent-raw-tag
+	  (semantic-tag-new-code $1 nil))))
+       (for_stmt
+	((FOR exprlist IN testlist COLON suite else_suite_pair_opt)
+	 (wisent-raw-tag
+	  (semantic-tag-new-code $1 nil))))
+       (try_stmt
+	((TRY COLON suite except_clause_suite_pair_list else_suite_pair_opt)
+	 (wisent-raw-tag
+	  (semantic-tag-new-code $1 nil)))
+	((TRY COLON suite FINALLY COLON suite)
+	 (wisent-raw-tag
+	  (semantic-tag-new-code $1 nil))))
+       (except_clause_suite_pair_list
+	((except_clause COLON suite)
+	 nil)
+	((except_clause_suite_pair_list except_clause COLON suite)
+	 nil))
+       (except_clause
+	((EXCEPT zero_one_or_two_test)
+	 nil))
+       (zero_one_or_two_test
+	(nil)
+	((test zero_or_one_comma_test)
+	 nil))
+       (funcdef
+	((DEF NAME function_parameter_list COLON suite)
+	 (wisent-raw-tag
+	  (semantic-tag-new-function $2 nil $3))))
+       (function_parameter_list
+	((PAREN_BLOCK)
+	 (let
+	     ((wisent-python-EXPANDING-block t))
+	   (semantic-parse-region
+	    (car $region1)
+	    (cdr $region1)
+	    'function_parameters 1))))
+       (function_parameters
+	((LPAREN)
+	 nil)
+	((RPAREN)
+	 nil)
+	((function_parameter COMMA))
+	((function_parameter RPAREN)))
+       (function_parameter
+	((fpdef_opt_test))
+	((MULT NAME)
+	 (wisent-raw-tag
+	  (semantic-tag-new-variable $2 nil nil)))
+	((EXPONENT NAME)
+	 (wisent-raw-tag
+	  (semantic-tag-new-variable $2 nil nil))))
+       (class_declaration
+	((CLASS NAME paren_class_list_opt COLON suite)
+	 (wisent-raw-tag
+	  (semantic-tag-new-type $2 $1 $5
+				 (cons $3 nil)))))
+       (paren_class_list_opt
+	(nil)
+	((paren_class_list)))
+       (paren_class_list
+	((PAREN_BLOCK)
+	 (let
+	     ((wisent-python-EXPANDING-block t))
+	   (mapcar 'semantic-tag-name
+		   (semantic-parse-region
+		    (car $region1)
+		    (cdr $region1)
+		    'paren_classes 1)))))
+       (paren_classes
+	((LPAREN)
+	 nil)
+	((RPAREN)
+	 nil)
+	((paren_class COMMA)
+	 (wisent-raw-tag
+	  (semantic-tag-new-variable $1 nil nil)))
+	((paren_class RPAREN)
+	 (wisent-raw-tag
+	  (semantic-tag-new-variable $1 nil nil))))
+       (paren_class
+	((dotted_name)))
+       (test
+	((test_test))
+	((lambdef)))
+       (test_test
+	((and_test))
+	((test_test OR and_test)
+	 nil))
+       (and_test
+	((not_test))
+	((and_test AND not_test)
+	 nil))
+       (not_test
+	((NOT not_test)
+	 nil)
+	((comparison)))
+       (comparison
+	((expr))
+	((comparison comp_op expr)
+	 nil))
+       (comp_op
+	((LT))
+	((GT))
+	((EQ))
+	((GE))
+	((LE))
+	((LTGT))
+	((NE))
+	((IN))
+	((NOT IN))
+	((IS))
+	((IS NOT)))
+       (expr
+	((xor_expr))
+	((expr BAR xor_expr)
+	 nil))
+       (xor_expr
+	((and_expr))
+	((xor_expr HAT and_expr)
+	 nil))
+       (and_expr
+	((shift_expr))
+	((and_expr AMP shift_expr)
+	 nil))
+       (shift_expr
+	((arith_expr))
+	((shift_expr shift_expr_operators arith_expr)
+	 nil))
+       (shift_expr_operators
+	((LTLT))
+	((GTGT)))
+       (arith_expr
+	((term))
+	((arith_expr plus_or_minus term)
+	 nil))
+       (plus_or_minus
+	((PLUS))
+	((MINUS)))
+       (term
+	((factor))
+	((term term_operator factor)
+	 nil))
+       (term_operator
+	((MULT))
+	((DIV))
+	((MOD))
+	((DIVDIV)))
+       (factor
+	((prefix_operators factor)
+	 nil)
+	((power)))
+       (prefix_operators
+	((PLUS))
+	((MINUS))
+	((TILDE)))
+       (power
+	((atom trailer_zom exponent_zom)
+	 (concat $1
+		 (if $2
+		     (concat " " $2 " ")
+		   "")
+		 (if $3
+		     (concat " " $3)
+		   ""))))
+       (trailer_zom
+	(nil)
+	((trailer_zom trailer)
+	 nil))
+       (exponent_zom
+	(nil)
+	((exponent_zom EXPONENT factor)
+	 nil))
+       (trailer
+	((PAREN_BLOCK)
+	 nil)
+	((BRACK_BLOCK)
+	 nil)
+	((PERIOD NAME)
+	 nil))
+       (atom
+	((PAREN_BLOCK)
+	 nil)
+	((BRACK_BLOCK)
+	 nil)
+	((BRACE_BLOCK)
+	 nil)
+	((BACKQUOTE testlist BACKQUOTE)
+	 nil)
+	((NAME))
+	((NUMBER_LITERAL))
+	((one_or_more_string)))
+       (test_list_opt
+	(nil)
+	((testlist)
+	 nil))
+       (testlist
+	((comma_sep_test_list comma_opt)))
+       (comma_sep_test_list
+	((test))
+	((comma_sep_test_list COMMA test)
+	 (format "%s, %s" $1 $3)))
+       (one_or_more_string
+	((STRING_LITERAL))
+	((one_or_more_string STRING_LITERAL)
+	 (concat $1 $2)))
+       (lambdef
+	((LAMBDA varargslist_opt COLON test)
+	 (format "%s %s" $1
+		 (or $2 ""))))
+       (varargslist_opt
+	(nil)
+	((varargslist)))
+       (varargslist
+	((fpdef_opt_test_list_comma_zom rest_args)
+	 (nconc $2 $1))
+	((fpdef_opt_test_list comma_opt)))
+       (rest_args
+	((MULT NAME multmult_name_opt)
+	 nil)
+	((EXPONENT NAME)
+	 nil))
+       (multmult_name_opt
+	(nil)
+	((COMMA EXPONENT NAME)
+	 (wisent-raw-tag
+	  (semantic-tag-new-variable $3 nil nil))))
+       (fpdef_opt_test_list_comma_zom
+	(nil)
+	((fpdef_opt_test_list_comma_zom fpdef_opt_test COMMA)
+	 (nconc $2 $1)))
+       (fpdef_opt_test_list
+	((fpdef_opt_test))
+	((fpdef_opt_test_list COMMA fpdef_opt_test)
+	 (nconc $3 $1)))
+       (fpdef_opt_test
+	((fpdef eq_test_opt)))
+       (fpdef
+	((NAME)
+	 (wisent-raw-tag
+	  (semantic-tag-new-variable $1 nil nil))))
+       (fplist
+	((fpdef_list comma_opt)))
+       (fpdef_list
+	((fpdef))
+	((fpdef_list COMMA fpdef)))
+       (eq_test_opt
+	(nil)
+	((ASSIGN test)
+	 nil))
+       (comma_opt
+	(nil)
+	((COMMA)))
+       (semicolon_opt
+	(nil)
+	((SEMICOLON))))
+     '(goal function_parameter paren_class indented_block function_parameters paren_classes indented_block_body)))
+  "Parser table.")
+
+(defun wisent-python-wy--install-parser ()
+  "Setup the Semantic Parser."
+  (semantic-install-function-overrides
+   '((parse-stream . wisent-parse-stream)))
+  (setq semantic-parser-name "LALR"
+	semantic--parse-table wisent-python-wy--parse-table
+	semantic-debug-parser-source "wisent-python.wy"
+	semantic-flex-keywords-obarray wisent-python-wy--keyword-table
+	semantic-lex-types-obarray wisent-python-wy--token-table)
+  ;; Collect unmatched syntax lexical tokens
+  (semantic-make-local-hook 'wisent-discarding-token-functions)
+  (add-hook 'wisent-discarding-token-functions
+	    'wisent-collect-unmatched-syntax nil t))
+
+
+;;; Analyzers
+
+(define-lex-keyword-type-analyzer wisent-python-wy--<keyword>-keyword-analyzer
+  "keyword analyzer for <keyword> tokens."
+  "\\(\\sw\\|\\s_\\)+")
+
+(define-lex-block-type-analyzer wisent-python-wy--<block>-block-analyzer
+  "block analyzer for <block> tokens."
+  "\\s(\\|\\s)"
+  '((("(" LPAREN PAREN_BLOCK)
+     ("{" LBRACE BRACE_BLOCK)
+     ("[" LBRACK BRACK_BLOCK))
+    (")" RPAREN)
+    ("}" RBRACE)
+    ("]" RBRACK))
+  )
+
+(define-lex-regex-type-analyzer wisent-python-wy--<symbol>-regexp-analyzer
+  "regexp analyzer for <symbol> tokens."
+  "\\(\\sw\\|\\s_\\)+"
+  nil
+  'NAME)
+
+(define-lex-regex-type-analyzer wisent-python-wy--<number>-regexp-analyzer
+  "regexp analyzer for <number> tokens."
+  semantic-lex-number-expression
+  nil
+  'NUMBER_LITERAL)
+
+(define-lex-string-type-analyzer wisent-python-wy--<punctuation>-string-analyzer
+  "string analyzer for <punctuation> tokens."
+  "\\(\\s.\\|\\s$\\|\\s'\\)+"
+  '((BACKQUOTE . "`")
+    (ASSIGN . "=")
+    (COMMA . ",")
+    (SEMICOLON . ";")
+    (COLON . ":")
+    (BAR . "|")
+    (TILDE . "~")
+    (PERIOD . ".")
+    (MINUS . "-")
+    (PLUS . "+")
+    (MOD . "%")
+    (DIV . "/")
+    (MULT . "*")
+    (AMP . "&")
+    (GT . ">")
+    (LT . "<")
+    (HAT . "^")
+    (NE . "!=")
+    (LTGT . "<>")
+    (HATEQ . "^=")
+    (OREQ . "|=")
+    (AMPEQ . "&=")
+    (MODEQ . "%=")
+    (DIVEQ . "/=")
+    (MULTEQ . "*=")
+    (MINUSEQ . "-=")
+    (PLUSEQ . "+=")
+    (LE . "<=")
+    (GE . ">=")
+    (EQ . "==")
+    (EXPONENT . "**")
+    (GTGT . ">>")
+    (LTLT . "<<")
+    (DIVDIV . "//")
+    (DIVDIVEQ . "//=")
+    (EXPEQ . "**=")
+    (GTGTEQ . ">>=")
+    (LTLTEQ . "<<="))
+  'punctuation)
+
+
+;;; Epilogue
+;;
+
+(provide 'semantic/wisent/python-wy)
+
+;;; semantic/wisent/python-wy.el ends here

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/lisp/cedet/semantic/wisent/python.el	Sat Jan 09 19:38:07 2010 -0500
@@ -0,0 +1,344 @@
+;;; wisent-python.el --- Semantic support for Python
+
+;; Copyright (C) 2002, 2004, 2006, 2007, 2008, 2009, 2010
+;; Free Software Foundation, Inc.
+
+;; Author: Richard Kim  <[e-mail]>
+;; Maintainer: Richard Kim  <[e-mail]>
+;; Created: June 2002
+;; Keywords: syntax
+
+;; This file is part of GNU Emacs.
+
+;; GNU Emacs is free software: you can redistribute it and/or modify
+;; it under the terms of the GNU General Public License as published by
+;; the Free Software Foundation, either version 3 of the License, or
+;; (at your option) any later version.
+
+;; GNU Emacs is distributed in the hope that it will be useful,
+;; but WITHOUT ANY WARRANTY; without even the implied warranty of
+;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+;; GNU General Public License for more details.
+
+;; You should have received a copy of the GNU General Public License
+;; along with GNU Emacs.  If not, see <http://www.gnu.org/licenses/>.
+
+;;; Commentary:
+;;
+;; Parser support for Python.
+
+;;; Code:
+
+(require 'semantic/wisent)
+(require 'semantic/wisent/python-wy)
+(require 'semantic/dep)
+(require 'semantic/ctxt)
+
+
+;;; Lexical analysis
+;;
+
+;; Python strings are delimited by either single quotes or double
+;; quotes, e.g., "I'm a string" and 'I too am s string'.
+;; In addition a string can have either a 'r' and/or 'u' prefix.
+;; The 'r' prefix means raw, i.e., normal backslash substitutions are
+;; to be suppressed.  For example, r"01\n34" is a string with six
+;; characters 0, 1, \, n, 3 and 4.  The 'u' prefix means the following
+;; string is a unicode.
+(defconst wisent-python-string-re
+  (concat (regexp-opt '("r" "u" "ur" "R" "U" "UR" "Ur" "uR") t)
+          "?['\"]")
+  "Regexp matching beginning of a python string.")
+
+(defvar wisent-python-EXPANDING-block nil
+  "Non-nil when expanding a paren block for Python lexical analyzer.")
+
+(defun wisent-python-implicit-line-joining-p ()
+  "Return non-nil if implicit line joining is active.
+That is, if inside an expressions in parentheses, square brackets or
+curly braces."
+  wisent-python-EXPANDING-block)
+
+(defsubst wisent-python-forward-string ()
+  "Move point at the end of the python string at point."
+  (when (looking-at wisent-python-string-re)
+     ;; skip the prefix
+    (and (match-end 1) (goto-char (match-end 1)))
+    ;; skip the quoted part
+    (cond
+     ((looking-at "\"\"\"[^\"]")
+      (search-forward "\"\"\"" nil nil 2))
+     ((looking-at "'''[^']")
+      (search-forward "'''" nil nil 2))
+     ((forward-sexp 1)))))
+
+(defun wisent-python-forward-line ()
+  "Move point to the beginning of the next logical line.
+Usually this is simply the next physical line unless strings,
+implicit/explicit line continuation, blank lines, or comment lines are
+encountered.  This function skips over such items so that the point is
+at the beginning of the next logical line.  If the current logical
+line ends at the end of the buffer, leave the point there."
+  (while (not (eolp))
+    (when (= (point)
+             (progn
+               (cond
+                ;; Skip over python strings.
+                ((looking-at wisent-python-string-re)
+                 (wisent-python-forward-string))
+                ;; At a comment start just goto end of line.
+                ((looking-at "\\s<")
+                 (end-of-line))
+                ;; Skip over generic lists and strings.
+                ((looking-at "\\(\\s(\\|\\s\"\\)")
+                 (forward-sexp 1))
+                ;; At the explicit line continuation character
+                ;; (backslash) move to next line.
+                ((looking-at "\\s\\")
+                 (forward-line 1))
+                ;; Skip over white space, word, symbol, punctuation,
+                ;; and paired delimiter (backquote) characters.
+                ((skip-syntax-forward "-w_.$)")))
+               (point)))
+      (error "python-forward-line endless loop detected")))
+  ;; The point is at eol, skip blank and comment lines.
+  (forward-comment (point-max))
+  ;; Goto the beginning of the next line.
+  (or (eobp) (beginning-of-line)))
+
+(defun wisent-python-forward-line-skip-indented ()
+  "Move point to the next logical line, skipping indented lines.
+That is the next line whose indentation is less than or equal to the
+identation of the current line."
+  (let ((indent (current-indentation)))
+    (while (progn (wisent-python-forward-line)
+                  (and (not (eobp))
+                       (> (current-indentation) indent))))))
+
+(defun wisent-python-end-of-block ()
+  "Move point to the end of the current block."
+  (let ((indent (current-indentation)))
+    (while (and (not (eobp)) (>= (current-indentation) indent))
+      (wisent-python-forward-line-skip-indented))
+    ;; Don't include final comments in current block bounds
+    (forward-comment (- (point-max)))
+    (or (bolp) (forward-line 1))
+    ))
+
+;; Indentation stack, what the Python (2.3) language spec. says:
+;;
+;; The indentation levels of consecutive lines are used to generate
+;; INDENT and DEDENT tokens, using a stack, as follows.
+;;
+;; Before the first line of the file is read, a single zero is pushed
+;; on the stack; this will never be popped off again. The numbers
+;; pushed on the stack will always be strictly increasing from bottom
+;; to top. At the beginning of each logical line, the line's
+;; indentation level is compared to the top of the stack. If it is
+;; equal, nothing happens. If it is larger, it is pushed on the stack,
+;; and one INDENT token is generated. If it is smaller, it must be one
+;; of the numbers occurring on the stack; all numbers on the stack
+;; that are larger are popped off, and for each number popped off a
+;; DEDENT token is generated. At the end of the file, a DEDENT token
+;; is generated for each number remaining on the stack that is larger
+;; than zero.
+(defvar wisent-python-indent-stack)
+
+(define-lex-analyzer wisent-python-lex-beginning-of-line
+  "Detect and create python indentation tokens at beginning of line."
+  (and
+   (bolp) (not (wisent-python-implicit-line-joining-p))
+   (let ((last-indent (car wisent-python-indent-stack))
+         (last-pos (point))
+         (curr-indent (current-indentation)))
+     (skip-syntax-forward "-")
+     (cond
+      ;; Skip comments and blank lines. No change in indentation.
+      ((or (eolp) (looking-at semantic-lex-comment-regex))
+       (forward-comment (point-max))
+       (or (eobp) (beginning-of-line))
+       (setq semantic-lex-end-point (point))
+       ;; Loop lexer to handle the next line.
+       t)
+      ;; No change in indentation.
+      ((= curr-indent last-indent)
+       (setq semantic-lex-end-point (point))
+       ;; Try next analyzers.
+       nil)
+      ;; Indentation increased
+      ((> curr-indent last-indent)
+       (if (or (not semantic-lex-maximum-depth)
+               (< semantic-lex-current-depth semantic-lex-maximum-depth))
+           (progn
+             ;; Return an INDENT lexical token
+             (setq semantic-lex-current-depth (1+ semantic-lex-current-depth))
+             (push curr-indent wisent-python-indent-stack)
+             (semantic-lex-push-token
+              (semantic-lex-token 'INDENT last-pos (point))))
+         ;; Add an INDENT_BLOCK token
+         (semantic-lex-push-token
+          (semantic-lex-token
+           'INDENT_BLOCK
+           (progn (beginning-of-line) (point))
+           (semantic-lex-unterminated-syntax-protection 'INDENT_BLOCK
+             (wisent-python-end-of-block)
+             (point)))))
+       ;; Loop lexer to handle tokens in current line.
+       t)
+      ;; Indentation decreased
+      (t
+       ;; Pop items from indentation stack
+       (while (< curr-indent last-indent)
+         (pop wisent-python-indent-stack)
+         (setq semantic-lex-current-depth (1- semantic-lex-current-depth)
+               last-indent (car wisent-python-indent-stack))
+         (semantic-lex-push-token
+          (semantic-lex-token 'DEDENT last-pos (point))))
+       ;; If pos did not change, then we must return nil so that
+       ;; other lexical analyzers can be run.
+       (/= last-pos (point))))))
+  ;; All the work was done in the above analyzer matching condition.
+  )
+
+(define-lex-regex-analyzer wisent-python-lex-end-of-line
+  "Detect and create python newline tokens.
+Just skip the newline character if the following line is an implicit
+continuation of current line."
+  "\\(\n\\|\\s>\\)"
+  (if (wisent-python-implicit-line-joining-p)
+      (setq semantic-lex-end-point (match-end 0))
+    (semantic-lex-push-token
+     (semantic-lex-token 'NEWLINE (point) (match-end 0)))))
+
+(define-lex-regex-analyzer wisent-python-lex-string
+  "Detect and create python string tokens."
+  wisent-python-string-re
+  (semantic-lex-push-token
+   (semantic-lex-token
+    'STRING_LITERAL
+    (point)
+    (semantic-lex-unterminated-syntax-protection 'STRING_LITERAL
+      (wisent-python-forward-string)
+      (point)))))
+
+(define-lex-regex-analyzer wisent-python-lex-ignore-backslash
+  "Detect and skip over backslash (explicit line joining) tokens.
+A backslash must be the last token of a physical line, it is illegal
+elsewhere on a line outside a string literal."
+  "\\s\\\\s-*$"
+  ;; Skip over the detected backslash and go to the first
+  ;; non-whitespace character in the next physical line.
+  (forward-line)
+  (skip-syntax-forward "-")
+  (setq semantic-lex-end-point (point)))
+
+(define-lex wisent-python-lexer
+  "Lexical Analyzer for Python code."
+  ;; Must analyze beginning of line first to handle indentation.
+  wisent-python-lex-beginning-of-line
+  wisent-python-lex-end-of-line
+  ;; Must analyze string before symbol to handle string prefix.
+  wisent-python-lex-string
+  ;; Analyzers auto-generated from grammar.
+  wisent-python-wy--<number>-regexp-analyzer
+  wisent-python-wy--<keyword>-keyword-analyzer
+  wisent-python-wy--<symbol>-regexp-analyzer
+  wisent-python-wy--<block>-block-analyzer
+  wisent-python-wy--<punctuation>-string-analyzer
+  ;; Ignored things.
+  wisent-python-lex-ignore-backslash
+  semantic-lex-ignore-whitespace
+  semantic-lex-ignore-comments
+  ;; Signal error on unhandled syntax.
+  semantic-lex-default-action)
+
+;;; Overridden Semantic API.
+;;
+(define-mode-local-override semantic-lex python-mode
+  (start end &optional depth length)
+  "Lexically analyze python code in current buffer.
+See the function `semantic-lex' for the meaning of the START, END,
+DEPTH and LENGTH arguments.
+This function calls `wisent-python-lexer' to actually perform the
+lexical analysis, then emits the necessary python DEDENT tokens from
+what remains in the `wisent-python-indent-stack'."
+  (let* ((wisent-python-indent-stack (list 0))
+         (stream (wisent-python-lexer start end depth length))
+         (semantic-lex-token-stream nil))
+    ;; Emit DEDENT tokens if something remains in the INDENT stack.
+    (while (> (pop wisent-python-indent-stack) 0)
+      (semantic-lex-push-token (semantic-lex-token 'DEDENT end end)))
+    (nconc stream (nreverse semantic-lex-token-stream))))
+
+(define-mode-local-override semantic-get-local-variables python-mode ()
+  "Get the local variables based on point's context.
+To be implemented for python!  For now just return nil."
+  nil)
+
+(defcustom-mode-local-semantic-dependency-system-include-path
+  python-mode semantic-python-dependency-system-include-path
+  nil
+  "The system include path used by Python langauge.")
+
+;;; Enable Semantic in `python-mode'.
+;;
+
+;;;###autoload
+(defun wisent-python-default-setup ()
+  "Setup buffer for parse."
+  (wisent-python-wy--install-parser)
+  (set (make-local-variable 'parse-sexp-ignore-comments) t)
+  (setq
+   ;; Character used to separation a parent/child relationship
+   semantic-type-relation-separator-character '(".")
+   semantic-command-separation-character ";"
+   ;; The following is no more necessary as semantic-lex is overriden
+   ;; in python-mode.
+   ;; semantic-lex-analyzer 'wisent-python-lexer
+
+   ;; Semantic to take over from the one provided by python.
+   ;; The python one, if it uses the senator advice, will hang
+   ;; Emacs unrecoverably.
+   imenu-create-index-function 'semantic-create-imenu-index
+
+   ;; I need a python guru to update this list:
+   semantic-symbol->name-assoc-list-for-type-parts '((variable . "Variables")
+						     (function . "Methods"))
+   semantic-symbol->name-assoc-list '((type . "Classes")
+				      (variable . "Variables")
+				      (function . "Functions")
+				      (include  . "Imports")
+				      (package  . "Package")
+				      (code . "Code")))
+   )
+
+;;;###autoload
+(add-hook 'python-mode-hook 'wisent-python-default-setup)
+
+;; Make sure the newer python modes pull in the same python
+;; mode overrides.
+(define-child-mode python-2-mode python-mode "Python 2 mode")
+(define-child-mode python-3-mode python-mode "Python 3 mode")
+
+
+;;; Test
+;;
+(defun wisent-python-lex-buffer ()
+  "Run `wisent-python-lexer' on current buffer."
+  (interactive)
+  (semantic-lex-init)
+  (let ((token-stream (semantic-lex (point-min) (point-max) 0)))
+    (with-current-buffer (get-buffer-create "*wisent-python-lexer*")
+      (erase-buffer)
+      (pp token-stream (current-buffer))
+      (goto-char (point-min))
+      (pop-to-buffer (current-buffer)))))
+
+(provide 'semantic/wisent/python)
+
+;; Local variables:
+;; generated-autoload-file: "../loaddefs.el"
+;; generated-autoload-load-name: "semantic/wisent/python"
+;; End:
+
+;;; semantic/wisent/python.el ends here