Added .vim/ folder to repo

From now on, the .vim/ folder is what will be maintained remotely.
This includes the .vimrc file.  Computers must now make a dummy simlink
pointing to this .vimrc file in their home directory.

 Changes to be committed:
	new file:   .vimrc
	new file:   Changelog.rst
	new file:   Makefile
	new file:   README.markdown
	new file:   README.rst
	new file:   after/plugin/snipMate.vim
	new file:   autoload/conque_term.vim
	new file:   autoload/pymode.vim
	new file:   autoload/pymode/breakpoint.vim
	new file:   autoload/pymode/doc.vim
	new file:   autoload/pymode/folding.vim
	new file:   autoload/pymode/indent.vim
	new file:   autoload/pymode/lint.vim
	new file:   autoload/pymode/motion.vim
	new file:   autoload/pymode/queue.vim
	new file:   autoload/pymode/run.vim
	new file:   autoload/pymode/troubleshooting.vim
	new file:   autoload/pymode/virtualenv.vim
	new file:   autoload/snipMate.vim
	new file:   colors/vividchalk.vim
	new file:   doc/conque_term.txt
	new file:   doc/pymode.txt
	new file:   doc/ropevim.txt
	new file:   doc/snipMate.txt
	new file:   doc/surround.txt
	new file:   doc/taglist.txt
	new file:   doc/tags
	new file:   ftplugin/html_snip_helper.vim
	new file:   ftplugin/python/pymode.vim
	new file:   plugin-info.txt
	new file:   plugin/conque_term.vim
	new file:   plugin/pymode.vim
	new file:   plugin/snipMate.vim
	new file:   plugin/surround.vim
	new file:   plugin/taglist.vim
	new file:   pylibs/__init__.py
	new file:   pylibs/__init__.pyc
	new file:   pylibs/autopep8.py
	new file:   pylibs/autopep8.pyc
	new file:   pylibs/logilab/__init__.py
	new file:   pylibs/logilab/astng/COPYING
	new file:   pylibs/logilab/astng/COPYING.LESSER
	new file:   pylibs/logilab/astng/README
	new file:   pylibs/logilab/astng/README.Python3
	new file:   pylibs/logilab/astng/__init__.py
	new file:   pylibs/logilab/astng/__pkginfo__.py
	new file:   pylibs/logilab/astng/as_string.py
	new file:   pylibs/logilab/astng/bases.py
	new file:   pylibs/logilab/astng/brain/py2stdlib.py
	new file:   pylibs/logilab/astng/builder.py
	new file:   pylibs/logilab/astng/exceptions.py
	new file:   pylibs/logilab/astng/inference.py
	new file:   pylibs/logilab/astng/inspector.py
	new file:   pylibs/logilab/astng/manager.py
	new file:   pylibs/logilab/astng/mixins.py
	new file:   pylibs/logilab/astng/node_classes.py
	new file:   pylibs/logilab/astng/nodes.py
	new file:   pylibs/logilab/astng/protocols.py
	new file:   pylibs/logilab/astng/raw_building.py
	new file:   pylibs/logilab/astng/rebuilder.py
	new file:   pylibs/logilab/astng/scoped_nodes.py
	new file:   pylibs/logilab/astng/utils.py
	new file:   pylibs/logilab/common/COPYING
	new file:   pylibs/logilab/common/COPYING.LESSER
	new file:   pylibs/logilab/common/README
	new file:   pylibs/logilab/common/README.Python3
	new file:   pylibs/logilab/common/__init__.py
	new file:   pylibs/logilab/common/__pkginfo__.py
	new file:   pylibs/logilab/common/announce.txt
	new file:   pylibs/logilab/common/cache.py
	new file:   pylibs/logilab/common/changelog.py
	new file:   pylibs/logilab/common/clcommands.py
	new file:   pylibs/logilab/common/cli.py
	new file:   pylibs/logilab/common/compat.py
	new file:   pylibs/logilab/common/configuration.py
	new file:   pylibs/logilab/common/contexts.py
	new file:   pylibs/logilab/common/corbautils.py
	new file:   pylibs/logilab/common/daemon.py
	new file:   pylibs/logilab/common/date.py
	new file:   pylibs/logilab/common/dbf.py
	new file:   pylibs/logilab/common/debugger.py
	new file:   pylibs/logilab/common/decorators.py
	new file:   pylibs/logilab/common/deprecation.py
	new file:   pylibs/logilab/common/fileutils.py
	new file:   pylibs/logilab/common/graph.py
	new file:   pylibs/logilab/common/hg.py
	new file:   pylibs/logilab/common/interface.py
	new file:   pylibs/logilab/common/logging_ext.py
	new file:   pylibs/logilab/common/modutils.py
	new file:   pylibs/logilab/common/optik_ext.py
	new file:   pylibs/logilab/common/optparser.py
	new file:   pylibs/logilab/common/pdf_ext.py
	new file:   pylibs/logilab/common/proc.py
	new file:   pylibs/logilab/common/pyro_ext.py
	new file:   pylibs/logilab/common/pytest.py
	new file:   pylibs/logilab/common/registry.py
	new file:   pylibs/logilab/common/shellutils.py
	new file:   pylibs/logilab/common/sphinx_ext.py
	new file:   pylibs/logilab/common/sphinxutils.py
	new file:   pylibs/logilab/common/table.py
	new file:   pylibs/logilab/common/tasksqueue.py
	new file:   pylibs/logilab/common/testlib.py
	new file:   pylibs/logilab/common/textutils.py
	new file:   pylibs/logilab/common/tree.py
	new file:   pylibs/logilab/common/umessage.py
	new file:   pylibs/logilab/common/ureports/__init__.py
	new file:   pylibs/logilab/common/ureports/docbook_writer.py
	new file:   pylibs/logilab/common/ureports/html_writer.py
	new file:   pylibs/logilab/common/ureports/nodes.py
	new file:   pylibs/logilab/common/ureports/text_writer.py
	new file:   pylibs/logilab/common/urllib2ext.py
	new file:   pylibs/logilab/common/vcgutils.py
	new file:   pylibs/logilab/common/visitor.py
	new file:   pylibs/logilab/common/xmlrpcutils.py
	new file:   pylibs/logilab/common/xmlutils.py
	new file:   pylibs/mccabe.py
	new file:   pylibs/mccabe.pyc
	new file:   pylibs/pep8.py
	new file:   pylibs/pep8.pyc
	new file:   pylibs/pyflakes/__init__.py
	new file:   pylibs/pyflakes/__init__.pyc
	new file:   pylibs/pyflakes/checker.py
	new file:   pylibs/pyflakes/checker.pyc
	new file:   pylibs/pyflakes/messages.py
	new file:   pylibs/pyflakes/messages.pyc
	new file:   pylibs/pylint/README
	new file:   pylibs/pylint/README.Python3
	new file:   pylibs/pylint/__init__.py
	new file:   pylibs/pylint/__pkginfo__.py
	new file:   pylibs/pylint/checkers/__init__.py
	new file:   pylibs/pylint/checkers/base.py
	new file:   pylibs/pylint/checkers/classes.py
	new file:   pylibs/pylint/checkers/design_analysis.py
	new file:   pylibs/pylint/checkers/exceptions.py
	new file:   pylibs/pylint/checkers/format.py
	new file:   pylibs/pylint/checkers/imports.py
	new file:   pylibs/pylint/checkers/logging.py
	new file:   pylibs/pylint/checkers/misc.py
	new file:   pylibs/pylint/checkers/newstyle.py
	new file:   pylibs/pylint/checkers/raw_metrics.py
	new file:   pylibs/pylint/checkers/similar.py
	new file:   pylibs/pylint/checkers/string_format.py
	new file:   pylibs/pylint/checkers/typecheck.py
	new file:   pylibs/pylint/checkers/utils.py
	new file:   pylibs/pylint/checkers/variables.py
	new file:   pylibs/pylint/config.py
	new file:   pylibs/pylint/interfaces.py
	new file:   pylibs/pylint/lint.py
	new file:   pylibs/pylint/reporters/__init__.py
	new file:   pylibs/pylint/reporters/guireporter.py
	new file:   pylibs/pylint/reporters/html.py
	new file:   pylibs/pylint/reporters/text.py
	new file:   pylibs/pylint/utils.py
	new file:   pylibs/pymode/__init__.py
	new file:   pylibs/pymode/__init__.pyc
	new file:   pylibs/pymode/auto.py
	new file:   pylibs/pymode/auto.pyc
	new file:   pylibs/pymode/interface.py
	new file:   pylibs/pymode/interface.pyc
	new file:   pylibs/pymode/lint.py
	new file:   pylibs/pymode/lint.pyc
	new file:   pylibs/pymode/queue.py
	new file:   pylibs/pymode/queue.pyc
	new file:   pylibs/rope/__init__.py
	new file:   pylibs/rope/__init__.pyc
	new file:   pylibs/rope/base/__init__.py
	new file:   pylibs/rope/base/__init__.pyc
	new file:   pylibs/rope/base/arguments.py
	new file:   pylibs/rope/base/arguments.pyc
	new file:   pylibs/rope/base/ast.py
	new file:   pylibs/rope/base/ast.pyc
	new file:   pylibs/rope/base/astutils.py
	new file:   pylibs/rope/base/astutils.pyc
	new file:   pylibs/rope/base/builtins.py
	new file:   pylibs/rope/base/builtins.pyc
	new file:   pylibs/rope/base/change.py
	new file:   pylibs/rope/base/change.pyc
	new file:   pylibs/rope/base/codeanalyze.py
	new file:   pylibs/rope/base/codeanalyze.pyc
	new file:   pylibs/rope/base/default_config.py
	new file:   pylibs/rope/base/default_config.pyc
	new file:   pylibs/rope/base/evaluate.py
	new file:   pylibs/rope/base/evaluate.pyc
	new file:   pylibs/rope/base/exceptions.py
	new file:   pylibs/rope/base/exceptions.pyc
	new file:   pylibs/rope/base/fscommands.py
	new file:   pylibs/rope/base/fscommands.pyc
	new file:   pylibs/rope/base/history.py
	new file:   pylibs/rope/base/history.pyc
	new file:   pylibs/rope/base/libutils.py
	new file:   pylibs/rope/base/libutils.pyc
	new file:   pylibs/rope/base/oi/__init__.py
	new file:   pylibs/rope/base/oi/__init__.pyc
	new file:   pylibs/rope/base/oi/doa.py
	new file:   pylibs/rope/base/oi/doa.pyc
	new file:   pylibs/rope/base/oi/memorydb.py
	new file:   pylibs/rope/base/oi/memorydb.pyc
	new file:   pylibs/rope/base/oi/objectdb.py
	new file:   pylibs/rope/base/oi/objectdb.pyc
	new file:   pylibs/rope/base/oi/objectinfo.py
	new file:   pylibs/rope/base/oi/objectinfo.pyc
	new file:   pylibs/rope/base/oi/runmod.py
	new file:   pylibs/rope/base/oi/soa.py
	new file:   pylibs/rope/base/oi/soa.pyc
	new file:   pylibs/rope/base/oi/soi.py
	new file:   pylibs/rope/base/oi/soi.pyc
	new file:   pylibs/rope/base/oi/transform.py
	new file:   pylibs/rope/base/oi/transform.pyc
	new file:   pylibs/rope/base/prefs.py
	new file:   pylibs/rope/base/prefs.pyc
	new file:   pylibs/rope/base/project.py
	new file:   pylibs/rope/base/project.pyc
	new file:   pylibs/rope/base/pycore.py
	new file:   pylibs/rope/base/pycore.pyc
	new file:   pylibs/rope/base/pynames.py
	new file:   pylibs/rope/base/pynames.pyc
	new file:   pylibs/rope/base/pynamesdef.py
	new file:   pylibs/rope/base/pynamesdef.pyc
	new file:   pylibs/rope/base/pyobjects.py
	new file:   pylibs/rope/base/pyobjects.pyc
	new file:   pylibs/rope/base/pyobjectsdef.py
	new file:   pylibs/rope/base/pyobjectsdef.pyc
	new file:   pylibs/rope/base/pyscopes.py
	new file:   pylibs/rope/base/pyscopes.pyc
	new file:   pylibs/rope/base/resourceobserver.py
	new file:   pylibs/rope/base/resourceobserver.pyc
	new file:   pylibs/rope/base/resources.py
	new file:   pylibs/rope/base/resources.pyc
	new file:   pylibs/rope/base/simplify.py
	new file:   pylibs/rope/base/simplify.pyc
	new file:   pylibs/rope/base/stdmods.py
	new file:   pylibs/rope/base/stdmods.pyc
	new file:   pylibs/rope/base/taskhandle.py
	new file:   pylibs/rope/base/taskhandle.pyc
	new file:   pylibs/rope/base/utils.py
	new file:   pylibs/rope/base/utils.pyc
	new file:   pylibs/rope/base/worder.py
	new file:   pylibs/rope/base/worder.pyc
	new file:   pylibs/rope/contrib/__init__.py
	new file:   pylibs/rope/contrib/__init__.pyc
	new file:   pylibs/rope/contrib/autoimport.py
	new file:   pylibs/rope/contrib/autoimport.pyc
	new file:   pylibs/rope/contrib/changestack.py
	new file:   pylibs/rope/contrib/codeassist.py
	new file:   pylibs/rope/contrib/codeassist.pyc
	new file:   pylibs/rope/contrib/finderrors.py
	new file:   pylibs/rope/contrib/findit.py
	new file:   pylibs/rope/contrib/findit.pyc
	new file:   pylibs/rope/contrib/fixmodnames.py
	new file:   pylibs/rope/contrib/fixsyntax.py
	new file:   pylibs/rope/contrib/fixsyntax.pyc
	new file:   pylibs/rope/contrib/generate.py
	new file:   pylibs/rope/contrib/generate.pyc
	new file:   pylibs/rope/refactor/__init__.py
	new file:   pylibs/rope/refactor/__init__.pyc
	new file:   pylibs/rope/refactor/change_signature.py
	new file:   pylibs/rope/refactor/change_signature.pyc
	new file:   pylibs/rope/refactor/encapsulate_field.py
	new file:   pylibs/rope/refactor/extract.py
	new file:   pylibs/rope/refactor/extract.pyc
	new file:   pylibs/rope/refactor/functionutils.py
	new file:   pylibs/rope/refactor/functionutils.pyc
	new file:   pylibs/rope/refactor/importutils/__init__.py
	new file:   pylibs/rope/refactor/importutils/__init__.pyc
	new file:   pylibs/rope/refactor/importutils/actions.py
	new file:   pylibs/rope/refactor/importutils/actions.pyc
	new file:   pylibs/rope/refactor/importutils/importinfo.py
	new file:   pylibs/rope/refactor/importutils/importinfo.pyc
	new file:   pylibs/rope/refactor/importutils/module_imports.py
	new file:   pylibs/rope/refactor/importutils/module_imports.pyc
	new file:   pylibs/rope/refactor/inline.py
	new file:   pylibs/rope/refactor/inline.pyc
	new file:   pylibs/rope/refactor/introduce_factory.py
	new file:   pylibs/rope/refactor/introduce_factory.pyc
	new file:   pylibs/rope/refactor/introduce_parameter.py
	new file:   pylibs/rope/refactor/localtofield.py
	new file:   pylibs/rope/refactor/method_object.py
	new file:   pylibs/rope/refactor/method_object.pyc
	new file:   pylibs/rope/refactor/move.py
	new file:   pylibs/rope/refactor/move.pyc
	new file:   pylibs/rope/refactor/multiproject.py
	new file:   pylibs/rope/refactor/occurrences.py
	new file:   pylibs/rope/refactor/occurrences.pyc
	new file:   pylibs/rope/refactor/patchedast.py
	new file:   pylibs/rope/refactor/patchedast.pyc
	new file:   pylibs/rope/refactor/rename.py
	new file:   pylibs/rope/refactor/rename.pyc
	new file:   pylibs/rope/refactor/restructure.py
	new file:   pylibs/rope/refactor/restructure.pyc
	new file:   pylibs/rope/refactor/similarfinder.py
	new file:   pylibs/rope/refactor/similarfinder.pyc
	new file:   pylibs/rope/refactor/sourceutils.py
	new file:   pylibs/rope/refactor/sourceutils.pyc
	new file:   pylibs/rope/refactor/suites.py
	new file:   pylibs/rope/refactor/suites.pyc
	new file:   pylibs/rope/refactor/topackage.py
	new file:   pylibs/rope/refactor/topackage.pyc
	new file:   pylibs/rope/refactor/usefunction.py
	new file:   pylibs/rope/refactor/usefunction.pyc
	new file:   pylibs/rope/refactor/wildcards.py
	new file:   pylibs/rope/refactor/wildcards.pyc
	new file:   pylibs/ropemode/__init__.py
	new file:   pylibs/ropemode/__init__.pyc
	new file:   pylibs/ropemode/decorators.py
	new file:   pylibs/ropemode/decorators.pyc
	new file:   pylibs/ropemode/dialog.py
	new file:   pylibs/ropemode/dialog.pyc
	new file:   pylibs/ropemode/environment.py
	new file:   pylibs/ropemode/environment.pyc
	new file:   pylibs/ropemode/filter.py
	new file:   pylibs/ropemode/filter.pyc
	new file:   pylibs/ropemode/interface.py
	new file:   pylibs/ropemode/interface.pyc
	new file:   pylibs/ropemode/refactor.py
	new file:   pylibs/ropemode/refactor.pyc
	new file:   pylibs/ropemode/tests/__init__.py
	new file:   pylibs/ropemode/tests/decorators_test.py
	new file:   pylibs/ropevim.py
	new file:   pylibs/ropevim.pyc
	new file:   pylint.ini
	new file:   snippets/_.snippets
	new file:   snippets/autoit.snippets
	new file:   snippets/c.snippets
	new file:   snippets/cpp.snippets
	new file:   snippets/erlang.snippets
	new file:   snippets/html.snippets
	new file:   snippets/java.snippets
	new file:   snippets/javascript.snippets
	new file:   snippets/mako.snippets
	new file:   snippets/objc.snippets
	new file:   snippets/perl.snippets
	new file:   snippets/php.snippets
	new file:   snippets/python.snippets
	new file:   snippets/ruby.snippets
	new file:   snippets/sh.snippets
	new file:   snippets/snippet.snippets
	new file:   snippets/tcl.snippets
	new file:   snippets/tex.snippets
	new file:   snippets/vim.snippets
	new file:   snippets/zsh.snippets
	new file:   syntax/conque_term.vim
	new file:   syntax/python.vim
	new file:   syntax/python3.0.vim
	new file:   syntax/snippet.vim

.vimrc

@@ -0,0 +1,51 @@
+" wreed vimrc
+" vim: set foldmethod=marker:
+
+
+"{{{ ***** VIM FEATURES ***** "
+" allow project-specific .vimrc files
+set exrc
+set secure
+" allow the use of a modeline
+set modeline
+" }}}
+
+"{{{ ***** VISUALS ***** "
+" set number
+set relativenumber
+" set foldmethod=syntax
+set foldcolumn=3
+
+set expandtab
+set shiftwidth=4
+set softtabstop=4
+set autoindent
+set smartindent
+set cindent
+filetype indent plugin on
+
+colorscheme koehler
+
+set showcmd
+"}}}
+
+"{{{ ***** KEY MAPPINGS ***** "
+set pastetoggle=<F2>
+nmap <F12> :mks!
+nmap <F4> :make!
+" }}}
+
+"{{{ ***** PROJECTS ***** "
+
+" set default project options
+set makeprg=rakefds
+" }}}
+
+"{{{ *****  PLUGINS  ***** "
+
+" PyMode options
+let pymode_lint_ignore="E501,E401,E225,W191,W391,W404"
+" use rope code assist instead of a complete function
+au FileType python inoremap <expr> <S-Space> '<C-r>=RopeCodeAssistInsertMode()<CR><C-r>=pumvisible() ? "\<lt>C-p>\<lt>Down>" : ""<CR>'
+" }}}
+

Changelog.rst

@@ -0,0 +1,221 @@
+Changelog
+=========
+
+## 2012-09-07 0.6.10
+--------------------
+* Dont raise an exception when Logger has no message handler (c) nixon
+* Improve performance of white space removal (c) Dave Smith
+* Improve ropemode support (c) s0undt3ch 
+* Add `g:pymode_updatetime` option
+* Update autopep8 to version 0.8.1
+
+## 2012-09-07 0.6.9
+-------------------
+* Update autopep8
+* Improve pymode#troubleshooting#Test()
+
+## 2012-09-06 0.6.8
+-------------------
+* Add PEP8 indentation ":help 'pymode_indent'" 
+
+## 2012-08-15 0.6.7
+-------------------
+* Fix documentation. Thanks (c) bgrant;
+* Fix pymode "async queue" support.
+
+## 2012-08-02 0.6.6
+-------------------
+* Updated Pep8 to version 1.3.3
+* Updated Pylint to version 0.25.2
+* Fixed virtualenv support for windows users
+* Added pymode modeline ':help PythonModeModeline'
+* Added diagnostic tool ':call pymode#troubleshooting#Test()'
+* Added `PyLintAuto` command ':help PyLintAuto' 
+* Code checking is async operation now
+* More, more fast the pymode folding
+* Repaired execution of python code
+
+## 2012-05-24 0.6.4
+-------------------
+* Add 'pymode_paths' option
+* Rope updated to version 0.9.4
+
+## 2012-04-18 0.6.3
+-------------------
+* Fix pydocs integration
+
+## 2012-04-10 0.6.2
+-------------------
+* Fix pymode_run for "unnamed" clipboard
+* Add 'pymode_lint_mccabe_complexity' option
+* Update Pep8 to version 1.0.1
+* Warning! Change 'pymode_rope_goto_def_newwin' option
+  for open "goto definition" in new window, set it to 'new' or 'vnew'
+  for horizontally or vertically split
+  If you use default behaviour (in the same buffer), not changes needed.
+
+## 2012-03-13 0.6.0
+-------------------
+* Add 'pymode_lint_hold' option 
+* Improve pymode loading speed
+* Add pep8, mccabe lint checkers
+* Now g:pymode_lint_checker can have many values
+  Ex. "pep8,pyflakes,mccabe"
+* Add 'pymode_lint_ignore' and 'pymode_lint_select' options 
+* Fix rope keys
+* Fix python motion in visual mode
+* Add folding 'pymode_folding'
+* Warning: 'pymode_lint_checker' now set to 'pyflakes,pep8,mccabe' by default
+
+## 2012-02-12 0.5.8
+-------------------
+* Fix pylint for Windows users
+* Python documentation search running from Vim (delete g:pydoc option)
+* Python code execution running from Vim (delete g:python option)
+
+## 2012-02-11 0.5.7
+-------------------
+* Fix 'g:pymode_lint_message' mode error
+* Fix breakpoints
+* Fix python paths and virtualenv detection
+
+## 2012-02-06 0.5.6
+-------------------
+* Fix 'g:pymode_syntax' option
+* Show error message in bottom part of screen
+  see 'g:pymode_lint_message'
+* Fix pylint for windows users
+* Fix breakpoint command (Use pdb when idpb not installed)
+
+## 2012-01-17 0.5.5
+-------------------
+* Add a sign for info messages from pylint.
+  (c) Fredrik Henrysson
+* Change motion keys: vic - viC, dam - daM and etc
+* Add 'g:pymode_lint_onfly' option
+
+## 2012-01-09 0.5.3
+-------------------
+* Prevent the configuration from breaking python-mode
+  (c) Dirk Wallenstein
+
+## 2012-01-08 0.5.2
+-------------------
+* Fix ropeomnicompletion
+* Add preview documentation
+
+## 2012-01-06 0.5.1
+-------------------
+* Happy new year!
+* Objects and motion  fixes
+
+## 2011-11-30 0.5.0
+-------------------
+* Add python objects and motions (beta)
+  :h pymode_motion
+
+## 2011-11-27 0.4.8
+-------------------
+* Add `PyLintWindowToggle` command
+* Fix some bugs
+
+## 2011-11-23 0.4.6
+-------------------
+* Enable all syntax highlighting
+  For old settings set in your vimrc:
+    let g:pymode_syntax_builtin_objs = 0
+    let g:pymode_syntax_builtin_funcs = 0
+
+* Change namespace of syntax variables
+  See README
+
+## 2011-11-18 0.4.5
+-------------------
+* Add 'g:pymode_syntax' option
+* Highlight 'self' keyword
+
+## 2011-11-16 0.4.4
+-------------------
+* Minor fixes
+
+## 2011-11-11 0.4.3
+-------------------
+* Fix pyflakes
+
+## 2011-11-09 0.4.2
+-------------------
+* Add FAQ
+* Some refactoring and fixes
+
+## 2011-11-08 0.4.0
+-------------------
+* Add alternative code checker "pyflakes"
+  See :h 'pymode_lint_checker'
+* Update install docs
+
+## 2011-10-30 0.3.3
+-------------------
+* Fix RopeShowDoc
+
+## 2011-10-28 0.3.2
+-------------------
+* Add 'g:pymode_options_*' stuff, for ability
+  to disable default pymode options for python buffers
+
+## 2011-10-27 0.3.1
+-------------------
+* Add 'g:pymode_rope_always_show_complete_menu' option
+* Some pylint fixes
+
+## 2011-10-25 0.3.0
+-------------------
+* Add g:pymode_lint_minheight and g:pymode_lint_maxheight
+  options
+* Fix PyLintToggle
+* Fix Rope and PyLint libs loading
+
+## 2011-10-21 0.2.12
+--------------------
+* Auto open cwindow with results
+  on rope find operations
+
+## 2011-10-20 0.2.11
+--------------------
+* Add 'pymode_lint_jump' option
+
+## 2011-10-19 0.2.10
+--------------------
+* Minor fixes (virtualenv loading, buffer commands)
+
+## 2011-10-18 0.2.6
+-------------------
+* Add <C-space> shortcut for macvim users.
+* Add VIRTUALENV support
+
+## 2011-10-17 0.2.4
+-------------------
+* Add current work path to sys.path
+* Add 'g:pymode' option (disable/enable pylint and rope)
+* Fix pylint copyright
+* Hotfix rope autocomplete
+
+## 2011-10-15 0.2.1
+-------------------
+* Change rope variables (ropevim_<name> -> pymode_rope_<name>)
+* Add "pymode_rope_auto_project" option (default: 1)
+* Update and fix docs
+* 'pymode_rope_extended_complete' set by default
+* Auto generate rope project and cache
+* "<C-c>r a" for RopeAutoImport
+
+## 2011-10-12 0.1.4
+-------------------
+* Add default pylint configuration
+
+## 2011-10-12 0.1.3
+-------------------
+* Fix pylint and update docs
+
+## 2011-10-11 0.1.2
+-------------------
+* First public release

Makefile

@@ -0,0 +1,3 @@
+.PHONY: clean
+clean:
+	find . -name "*.pyc" -delete

README.markdown

@@ -0,0 +1,5 @@
+Quickly install with:
+
+    git clone git://github.com/msanders/snipmate.vim.git
+	cd snipmate.vim
+	cp -R * ~/.vim

README.rst

@@ -0,0 +1,518 @@
+Python-mode, Python in VIM
+##########################
+
+Python-mode is a vim plugin that allows you to use the pylint_, rope_, pydoc_, pyflakes_, pep8_, mccabe_ libraries in vim to provide
+features like python code looking for bugs, refactoring and some other useful things.
+
+This plugin allow you create python code in vim very easily.
+There is no need to install the pylint_, rope_ or any used python library on your system.
+
+- Python objects and motion (]], 3[[, ]]M, vaC, viM, daC, ciM, ...)
+- Folding of python code
+- Virtualenv support
+- Highlight syntax errors
+- Highlight and auto fix unused imports
+- Many linters (pylint_, pyflakes_, ...) that can be run simultaneously
+- Strong code completion
+- Code refactoring
+- Python documentation
+- Run python code
+- Go to definition
+- Powerful customization
+- And more, more ...
+
+See (very old) screencast here: http://t.co/3b0bzeXA (sorry for quality, this is my first screencast)
+
+
+.. contents::
+
+
+Changelog
+=========
+
+## 2012-08-02 0.6.5
+-------------------
+* Updated Pep8 to version 1.3.3
+* Updated Pylint to version 0.25.2
+* Fixed virtualenv support for windows users
+* Added pymode modeline ':help PythonModeModeline'
+* Added diagnostic tool ':call pymode#troubleshooting#Test()'
+* Added `PyLintAuto` command ':help PyLintAuto' 
+* Code checking is async operation now
+* More, more fast the pymode folding
+* Repaired execution of python code
+
+
+Requirements
+============
+
+- VIM >= 7.0 with python support
+  (also ``--with-features=big`` if you want use g:pymode_lint_signs)
+
+
+
+How to install
+==============
+
+
+Using pathogen_ (recomended)
+----------------------------
+::
+
+    % cd ~/.vim
+    % mkdir -p bundle && cd bundle
+    % git clone git://github.com/klen/python-mode.git
+
+- Enable pathogen_ in your ``~/.vimrc``: ::
+
+    " Pathogen load
+    filetype off
+
+    call pathogen#infect()
+    call pathogen#helptags()
+
+    filetype plugin indent on
+    syntax on
+
+
+Manually
+--------
+::
+
+    % git clone git://github.com/klen/python-mode.git
+    % cd python-mode
+    % cp -R * ~/.vim
+
+Then rebuild **helptags** in vim::
+
+    :helptags ~/.vim/doc/
+
+
+.. note:: **filetype-plugin** (``:help filetype-plugin-on``) and **filetype-indent** (``:help filetype-indent-on``)
+    must be enabled for use python-mode.
+
+
+Troubleshooting
+===============
+
+If your python-mode dont work, type command: ::
+
+    :call pymode#troubleshooting#Test()
+
+And fix warnings or copy output and send it to me (ex. with github issue).
+
+
+Settings
+========
+
+.. note:: Also you can see vim help. ``:help PythonModeOptions``
+
+To change this settings, edit your ``~/.vimrc``: ::
+
+    " Disable pylint checking every save
+    let g:pymode_lint_write = 0
+
+    " Set key 'R' for run python code
+    let g:pymode_run_key = 'R'
+
+
+Show documentation
+------------------
+
+Default values: ::
+
+    " Load show documentation plugin
+    let g:pymode_doc = 1
+
+    " Key for show python documentation
+    let g:pymode_doc_key = 'K'
+
+
+Run python code
+---------------
+
+Default values: ::
+
+    " Load run code plugin
+    let g:pymode_run = 1
+
+    " Key for run python code
+    let g:pymode_run_key = '<leader>r'
+
+
+Code checking
+-------------
+
+Default values: ::
+
+    " Load pylint code plugin
+    let g:pymode_lint = 1
+
+    " Switch pylint, pyflakes, pep8, mccabe code-checkers
+    " Can have multiply values "pep8,pyflakes,mcccabe"
+    let g:pymode_lint_checker = "pyflakes,pep8,mccabe"
+
+    " Skip errors and warnings
+    " E.g. "E501,W002", "E2,W" (Skip all Warnings and Errors startswith E2) and etc
+    let g:pymode_lint_ignore = "E501"
+
+    " Select errors and warnings
+    " E.g. "E4,W"
+    let g:pymode_lint_select = ""
+
+    " Run linter on the fly
+    let g:pymode_lint_onfly = 0
+
+    " Pylint configuration file
+    " If file not found use 'pylintrc' from python-mode plugin directory
+    let g:pymode_lint_config = "$HOME/.pylintrc"
+
+    " Check code every save
+    let g:pymode_lint_write = 1
+
+    " Auto open cwindow if errors be finded
+    let g:pymode_lint_cwindow = 1
+
+    " Show error message if cursor placed at the error line
+    let g:pymode_lint_message = 1
+
+    " Auto jump on first error
+    let g:pymode_lint_jump = 0
+
+    " Hold cursor in current window
+    " when quickfix is open
+    let g:pymode_lint_hold = 0
+
+    " Place error signs
+    let g:pymode_lint_signs = 1
+
+    " Maximum allowed mccabe complexity
+    let g:pymode_lint_mccabe_complexity = 8
+
+    " Minimal height of pylint error window
+    let g:pymode_lint_minheight = 3
+
+    " Maximal height of pylint error window
+    let g:pymode_lint_maxheight = 6
+
+
+.. note:: 
+    Pylint options (ex. disable messages) may be defined in ``$HOME/pylint.rc``
+    See pylint documentation: http://pylint-messages.wikidot.com/all-codes
+
+
+Rope refactoring library
+------------------------
+
+Default values: ::
+
+    " Load rope plugin
+    let g:pymode_rope = 1
+
+    " Auto create and open ropeproject
+    let g:pymode_rope_auto_project = 1
+
+    " Enable autoimport
+    let g:pymode_rope_enable_autoimport = 1
+
+    " Auto generate global cache
+    let g:pymode_rope_autoimport_generate = 1
+
+    let g:pymode_rope_autoimport_underlineds = 0
+
+    let g:pymode_rope_codeassist_maxfixes = 10
+
+    let g:pymode_rope_sorted_completions = 1
+
+    let g:pymode_rope_extended_complete = 1
+
+    let g:pymode_rope_autoimport_modules = ["os","shutil","datetime"]
+
+    let g:pymode_rope_confirm_saving = 1
+
+    let g:pymode_rope_global_prefix = "<C-x>p"
+
+    let g:pymode_rope_local_prefix = "<C-c>r"
+
+    let g:pymode_rope_vim_completion = 1
+
+    let g:pymode_rope_guess_project = 1
+
+    let g:pymode_rope_goto_def_newwin = ""
+
+    let g:pymode_rope_always_show_complete_menu = 0
+
+
+Automatically folding of python code
+--------------------------------------
+
+Default values: ::
+
+    " Enable python folding
+    let g:pymode_folding = 0
+
+
+Vim python motions and operators
+--------------------------------
+
+Default values: ::
+
+    " Enable python objects and motion
+    let g:pymode_motion = 1
+
+
+Virtualenv support
+------------------
+
+Default values: ::
+
+    " Auto fix vim python paths if virtualenv enabled
+    let g:pymode_virtualenv = 1
+
+
+Other stuff
+-----------
+
+Default values: ::
+
+    " Additional python paths
+    let g:pymode_paths = []
+
+    " Load breakpoints plugin
+    let g:pymode_breakpoint = 1
+
+    " Key for set/unset breakpoint
+    let g:pymode_breakpoint_key = '<leader>b'
+
+    " Autoremove unused whitespaces
+    let g:pymode_utils_whitespaces = 1
+
+    " Enable pymode indentation
+    let g:pymode_indent = 1
+
+    " Set default pymode python options
+    let g:pymode_options = 1
+
+
+Syntax highlight
+----------------
+
+Default values: ::
+
+    " Enable pymode's custom syntax highlighting
+    let g:pymode_syntax = 1
+
+    " Enable all python highlightings
+    let g:pymode_syntax_all = 1
+
+    " Highlight "print" as function
+    let g:pymode_syntax_print_as_function = 0
+
+    " Highlight indentation errors
+    let g:pymode_syntax_indent_errors = g:pymode_syntax_all
+
+    " Highlight trailing spaces
+    let g:pymode_syntax_space_errors = g:pymode_syntax_all
+
+    " Highlight string formatting
+    let g:pymode_syntax_string_formatting = g:pymode_syntax_all
+
+    " Highlight str.format syntax
+    let g:pymode_syntax_string_format = g:pymode_syntax_all
+
+    " Highlight string.Template syntax
+    let g:pymode_syntax_string_templates = g:pymode_syntax_all
+
+    " Highlight doc-tests
+    let g:pymode_syntax_doctests = g:pymode_syntax_all
+
+    " Highlight builtin objects (__doc__, self, etc)
+    let g:pymode_syntax_builtin_objs = g:pymode_syntax_all
+
+    " Highlight builtin functions
+    let g:pymode_syntax_builtin_funcs = g:pymode_syntax_all
+
+    " Highlight exceptions
+    let g:pymode_syntax_highlight_exceptions = g:pymode_syntax_all
+
+    " For fast machines
+    let g:pymode_syntax_slow_sync = 0
+
+
+Default keys
+============
+
+.. note:: Also you can see vim help ``:help PythonModeKeys``
+
+============== =============
+Keys           Command
+============== =============
+**K**          Show python docs (g:pymode_doc enabled)
+-------------- -------------
+**<C-Space>**  Rope autocomplete (g:pymode_rope enabled)
+-------------- -------------
+**<C-c>g**     Rope goto definition  (g:pymode_rope enabled)
+-------------- -------------
+**<C-c>d**     Rope show documentation  (g:pymode_rope enabled)
+-------------- -------------
+**<C-c>f**     Rope find occurrences  (g:pymode_rope enabled)
+-------------- -------------
+**<Leader>r**  Run python  (g:pymode_run enabled)
+-------------- -------------
+**<Leader>b**  Set, unset breakpoint (g:pymode_breakpoint enabled)
+-------------- -------------
+[[             Jump on previous class or function (normal, visual, operator modes)
+-------------- -------------
+]]             Jump on next class or function  (normal, visual, operator modes)
+-------------- -------------
+[M             Jump on previous class or method (normal, visual, operator modes)
+-------------- -------------
+]M             Jump on next class or method (normal, visual, operator modes)
+-------------- -------------
+aC C           Select a class. Ex: vaC, daC, dC, yaC, yC, caC, cC (normal, operator modes)
+-------------- -------------
+iC             Select inner class. Ex: viC, diC, yiC, ciC (normal, operator modes)
+-------------- -------------
+aM M           Select a function or method. Ex: vaM, daM, dM, yaM, yM, caM, cM (normal, operator modes)
+-------------- -------------
+iM             Select inner function or method. Ex: viM, diM, yiM, ciM (normal, operator modes)
+============== =============
+
+.. note:: See also ``:help ropevim.txt``
+
+
+Commands
+========
+
+.. note:: Also you can see vim help ``:help PythonModeCommands``
+
+==================== =============
+Command              Description
+==================== =============
+:Pydoc <args>        Show python documentation
+-------------------- -------------
+PyLintToggle         Enable, disable pylint
+-------------------- -------------
+PyLintCheckerToggle  Toggle code checker (pylint, pyflakes)
+-------------------- -------------
+PyLint               Check current buffer
+-------------------- -------------
+PyLintAuto           Automatic fix PEP8 errors
+-------------------- -------------
+Pyrun                Run current buffer in python
+==================== =============
+
+.. note:: See also ``:help ropevim.txt``
+
+
+F.A.Q.
+======
+
+Rope completion is very slow
+----------------------------
+
+To work rope_ creates a service directory: ``.ropeproject``.
+If ``g:pymode_rope_guess_project`` set (by default) and ``.ropeproject`` in current dir not found, rope scan ``.ropeproject`` on every dir in parent path.
+If rope finded ``.ropeproject`` in parent dirs, rope set project for all child dir and scan may be slow for many dirs and files.
+
+Solutions:
+
+- Disable ``g:pymode_rope_guess_project`` to make rope always create ``.ropeproject`` in current dir.
+- Delete ``.ropeproject`` from dip parent dir to make rope create ``.ropeproject`` in current dir.
+- Press ``<C-x>po`` or ``:RopeOpenProject`` to make force rope create ``.ropeproject`` in current dir.
+
+
+
+Pylint check is very slow
+-------------------------
+
+In some projects pylint_ may check slowly, because it also scan imported modules if posible.
+Try use pyflakes_, see ``:h 'pymode_lint_checker'``.
+
+.. note:: You may ``set exrc`` and ``set secure`` in your ``vimrc`` for auto set custom settings from ``.vimrc`` from your projects directories.
+    Example: On Flask projects I automaticly set ``g:pymode_lint_checker = "pyflakes"``, on django ``g:pymode_lint_cheker = "pylint"``
+
+
+OSX cannot import urandom
+-------------------------
+
+See: https://groups.google.com/forum/?fromgroups=#!topic/vim_dev/2NXKF6kDONo
+
+The sequence of commands that fixed this: ::
+
+    brew unlink python
+    brew unlink macvim
+    brew remove macvim
+    brew install -v --force macvim
+    brew link macvim
+    brew link python
+
+
+Bugtracker
+===========
+
+If you have any suggestions, bug reports or
+annoyances please report them to the issue tracker
+at https://github.com/klen/python-mode/issues
+
+
+Contributing
+============
+
+Development of pylint-mode happens at github: https://github.com/klen/python-mode
+
+
+Copyright
+=========
+
+Copyright (C) 2012 Kirill Klenov (klen_)
+
+    **Rope**
+        Copyright (C) 2006-2010 Ali Gholami Rudi
+
+        Copyright (C) 2009-2010 Anton Gritsay
+
+    **Pylint**
+        Copyright (C) 2003-2011 LOGILAB S.A. (Paris, FRANCE).
+        http://www.logilab.fr/
+
+    **Pyflakes**:
+        Copyright (c) 2005 Divmod, Inc.
+        http://www.divmod.com/
+
+    **PEP8**
+        Copyright (C) 2006 Johann C. Rocholl <johann@rocholl.net>
+        http://github.com/jcrocholl/pep8
+
+    **autopep8**:
+        Copyright (c) 2012 hhatto <hhatto.jp@gmail.com>
+        https://github.com/hhatto/autopep8
+
+    **Python syntax for vim**
+        Copyright (c) 2010 Dmitry Vasiliev
+        http://www.hlabs.spb.ru/vim/python.vim
+
+    **PEP8 VIM indentation**
+        Copyright (c) 2012 Hynek Schlawack <hs@ox.cx>
+        http://github.com/hynek/vim-python-pep8-indent
+
+
+License
+=======
+
+Licensed under a `GNU lesser general public license`_.
+
+If you like this plugin, you can send me postcard :) 
+My address is here: "Russia, 143401, Krasnogorsk, Shkolnaya 1-19" to "Kirill Klenov".
+**Thanks for support!**
+
+
+.. _GNU lesser general public license: http://www.gnu.org/copyleft/lesser.html
+.. _klen: http://klen.github.com/
+.. _pylint: http://www.logilab.org/857
+.. _pyflakes: http://pypi.python.org/pypi/pyflakes
+.. _rope: http://rope.sourceforge.net/
+.. _pydoc: http://docs.python.org/library/pydoc.html
+.. _pathogen: https://github.com/tpope/vim-pathogen
+.. _pep8: http://pypi.python.org/pypi/pep8
+.. _mccabe: http://en.wikipedia.org/wiki/Cyclomatic_complexity

after/plugin/snipMate.vim

@@ -0,0 +1,40 @@
+" These are the mappings for snipMate.vim. Putting it here ensures that it
+" will be mapped after other plugins such as supertab.vim.
+if !exists('loaded_snips') || exists('s:did_snips_mappings')
+	finish
+endif
+let s:did_snips_mappings = 1
+
+" This is put here in the 'after' directory in order for snipMate to override
+" other plugin mappings (e.g., supertab).
+"
+" You can safely adjust these mappings to your preferences (as explained in
+" :help snipMate-remap).
+ino <silent> <tab> <c-r>=TriggerSnippet()<cr>
+snor <silent> <tab> <esc>i<right><c-r>=TriggerSnippet()<cr>
+ino <silent> <s-tab> <c-r>=BackwardsSnippet()<cr>
+snor <silent> <s-tab> <esc>i<right><c-r>=BackwardsSnippet()<cr>
+ino <silent> <c-r><tab> <c-r>=ShowAvailableSnips()<cr>
+
+" The default mappings for these are annoying & sometimes break snipMate.
+" You can change them back if you want, I've put them here for convenience.
+snor <bs> b<bs>
+snor <right> <esc>a
+snor <left> <esc>bi
+snor ' b<bs>'
+snor ` b<bs>`
+snor % b<bs>%
+snor U b<bs>U
+snor ^ b<bs>^
+snor \ b<bs>\
+snor <c-x> b<bs><c-x>
+
+" By default load snippets in snippets_dir
+if empty(snippets_dir)
+	finish
+endif
+
+call GetSnippets(snippets_dir, '_') " Get global snippets
+
+au FileType * if &ft != 'help' | call GetSnippets(snippets_dir, &ft) | endif
+" vim:noet:sw=4:ts=4:ft=vim

autoload/conque_term.vim

@@ -0,0 +1 @@
+/usr/share/vim-conque/autoload/conque_term.vim

autoload/pymode.vim

@@ -0,0 +1,190 @@
+" Python-mode base functions
+
+
+fun! pymode#Default(name, default) "{{{
+    " DESC: Set default value if it not exists
+    "
+    if !exists(a:name)
+        let {a:name} = a:default
+        return 0
+    endif
+    return 1
+endfunction "}}}
+
+
+fun! pymode#Option(name) "{{{
+
+    let name = 'b:pymode_' . a:name
+    if exists(name)
+        return eval(name)
+    endif
+
+    let name = 'g:pymode_' . a:name
+    return eval(name)
+
+endfunction "}}}
+
+
+fun! pymode#QuickfixOpen(onlyRecognized, holdCursor, maxHeight, minHeight, jumpError) "{{{
+    " DESC: Open quickfix window
+    "
+    let numErrors = len(filter(getqflist(), 'v:val.valid'))
+    let numOthers = len(getqflist()) - numErrors
+    if numErrors > 0 || (!a:onlyRecognized && numOthers > 0)
+        botright copen
+        exe max([min([line("$"), a:maxHeight]), a:minHeight]) . "wincmd _"
+        if a:jumpError
+            cc
+        elseif !a:holdCursor
+            wincmd p
+        endif
+    else
+        cclose
+    endif
+    redraw
+    if numOthers > 0
+        echo printf('Quickfix: %d(+%d)', numErrors, numOthers)
+    else
+        echo printf('Quickfix: %d', numErrors)
+    endif
+endfunction "}}}
+
+
+fun! pymode#PlaceSigns() "{{{
+    " DESC: Place error signs
+    "
+    if has('signs')
+        sign unplace *
+
+        if !pymode#Default("g:pymode_lint_signs_always_visible", 0) || g:pymode_lint_signs_always_visible
+            call RopeShowSignsRulerIfNeeded()
+        endif
+
+        for item in filter(getqflist(), 'v:val.bufnr != ""')
+            execute printf('silent! sign place 1 line=%d name=%s buffer=%d', item.lnum, item.type, item.bufnr)
+        endfor
+
+    endif
+endfunction "}}}
+
+
+fun! pymode#CheckProgram(name, append) "{{{
+    " DESC: Check program is executable or redifined by user.
+    "
+    let name = 'g:' . a:name
+    if pymode#Default(name, a:name)
+        return 1
+    endif
+    if !executable(eval(l:name))
+        echoerr "Can't find '".eval(name)."'. Please set ".name .", or extend $PATH, ".a:append
+        return 0
+    endif
+    return 1
+endfunction "}}}
+
+
+fun! pymode#TempBuffer() "{{{
+    " DESC: Open temp buffer.
+    "
+    pclose | botright 8new
+    setlocal buftype=nofile bufhidden=delete noswapfile nowrap previewwindow
+    redraw
+endfunction "}}}
+
+
+fun! pymode#ShowStr(str) "{{{
+    " DESC: Open temp buffer with `str`.
+    "
+    let g:pymode_curbuf = bufnr("%")
+    call pymode#TempBuffer()
+    put! =a:str
+    wincmd p
+    redraw
+endfunction "}}}
+
+
+fun! pymode#ShowCommand(cmd) "{{{
+    " DESC: Run command and open temp buffer with result
+    "
+    call pymode#TempBuffer()
+    try
+        silent exec 'r!' . a:cmd
+    catch /.*/
+        close
+        echoerr 'Command fail: '.a:cmd
+    endtry
+    redraw
+    normal gg
+    wincmd p
+endfunction "}}}
+
+
+fun! pymode#WideMessage(msg) "{{{
+    " DESC: Show wide message
+
+    let x=&ruler | let y=&showcmd
+    set noruler noshowcmd
+    redraw
+    echohl Debug | echo strpart(a:msg, 0, &columns-1) | echohl none
+    let &ruler=x | let &showcmd=y
+endfunction "}}}
+
+
+fun! pymode#BlockStart(lnum, ...) "{{{
+    let pattern = a:0 ? a:1 : '^\s*\(@\|class\s.*:\|def\s\)'
+    let lnum = a:lnum + 1
+    let indent = 100
+    while lnum
+        let lnum = prevnonblank(lnum - 1)
+        let test = indent(lnum)
+        let line = getline(lnum)
+        if line =~ '^\s*#' " Skip comments
+            continue
+        elseif !test " Zero-level regular line
+            return lnum
+        elseif test >= indent " Skip deeper or equal lines
+            continue
+        " Indent is strictly less at this point: check for def/class
+        elseif line =~ pattern && line !~ '^\s*@'
+            return lnum
+        endif
+        let indent = indent(lnum)
+    endwhile
+    return 0
+endfunction "}}}
+
+
+fun! pymode#BlockEnd(lnum, ...) "{{{
+    let indent = a:0 ? a:1 : indent(a:lnum)
+    let lnum = a:lnum
+    while lnum
+        let lnum = nextnonblank(lnum + 1)
+        if getline(lnum) =~ '^\s*#' | continue
+        elseif lnum && indent(lnum) <= indent
+            return lnum - 1
+        endif
+    endwhile
+    return line('$')
+endfunction "}}}
+
+
+fun! pymode#Modeline() "{{{
+    let modeline = getline(prevnonblank('$'))
+    if modeline =~ '^#\s\+pymode:'
+        for ex in split(modeline, ':')[1:]
+            let [name, value] = split(ex, '=')
+            let {'b:pymode_'.name} = value
+        endfor
+    endif
+    au BufRead <buffer> call pymode#Modeline()
+endfunction "}}}
+
+
+fun! pymode#TrimWhiteSpace() "{{{
+    let cursor_pos = getpos('.')
+    silent! %s/\s\+$//
+    call setpos('.', cursor_pos)
+endfunction "}}}
+
+
+" vim: fdm=marker:fdl=0

autoload/pymode/breakpoint.vim

@@ -0,0 +1,20 @@
+fun! pymode#breakpoint#Set(lnum) "{{{
+    let line = getline(a:lnum)
+    if strridx(line, g:pymode_breakpoint_cmd) != -1
+        normal dd
+    else
+        let plnum = prevnonblank(a:lnum)
+        call append(line('.')-1, repeat(' ', indent(plnum)).g:pymode_breakpoint_cmd)
+        normal k
+    endif
+
+    " Disable lint
+    let pymode_lint = g:pymode_lint
+    let g:pymode_lint = 0
+
+    " Save file
+    if &modifiable && &modified | write | endif	
+
+    let g:pymode_lint = pymode_lint
+
+endfunction "}}}

autoload/pymode/doc.vim

@@ -0,0 +1,19 @@
+" Python-mode search by documentation
+
+
+fun! pymode#doc#Show(word) "{{{
+    if a:word == ''
+        echoerr "No name/symbol under cursor!"
+    else
+        py import StringIO
+        py sys.stdout, _ = StringIO.StringIO(), sys.stdout
+        py help(vim.eval('a:word'))
+        py sys.stdout, out = _, sys.stdout.getvalue()
+        call pymode#TempBuffer()
+        py vim.current.buffer.append(out.split('\n'), 0)
+        wincmd p
+    endif
+endfunction "}}}
+
+
+" vim: fdm=marker:fdl=0

autoload/pymode/folding.vim

@@ -0,0 +1,60 @@
+" Python-mode folding functions
+
+
+let s:blank_regex = '^\s*$'
+let s:def_regex = '^\s*\(class\|def\) \w\+'
+
+
+fun! pymode#folding#text() " {{{
+    let fs = v:foldstart
+    while getline(fs) =~ '^\s*@' 
+        let fs = nextnonblank(fs + 1)
+    endwhile
+    let line = getline(fs)
+
+    let nucolwidth = &fdc + &number * &numberwidth
+    let windowwidth = winwidth(0) - nucolwidth - 3
+    let foldedlinecount = v:foldend - v:foldstart
+
+    " expand tabs into spaces
+    let onetab = strpart('          ', 0, &tabstop)
+    let line = substitute(line, '\t', onetab, 'g')
+
+    let line = strpart(line, 0, windowwidth - 2 -len(foldedlinecount))
+    let fillcharcount = windowwidth - len(line) - len(foldedlinecount)
+    return line . '…' . repeat(" ",fillcharcount) . foldedlinecount . '…' . ' '
+endfunction "}}}
+
+
+fun! pymode#folding#expr(lnum) "{{{
+
+    let line = getline(a:lnum)
+    let indent = indent(a:lnum)
+
+    if line =~ s:def_regex
+        return ">".(indent / &shiftwidth + 1)
+    endif
+
+    if line =~ '^\s*@'
+        return -1
+    endif
+
+    if line =~ s:blank_regex
+        let prev_line = getline(a:lnum - 1)
+        if prev_line =~ s:blank_regex
+            return -1
+        else
+            return foldlevel(prevnonblank(a:lnum))
+        endif
+    endif
+
+    if indent == 0
+        return 0
+    endif
+
+    return '='
+
+endfunction "}}}
+
+
+" vim: fdm=marker:fdl=0

autoload/pymode/indent.vim

@@ -0,0 +1,184 @@
+" PEP8 compatible Python indent file
+" Language:         Python
+" Maintainer:       Hynek Schlawack <hs@ox.cx>
+" Prev Maintainer:  Eric Mc Sween <em@tomcom.de> (address invalid)
+" Original Author:  David Bustos <bustos@caltech.edu> (address invalid)
+" Last Change:      2012-06-21
+" License:          Public Domainlet
+
+
+function! pymode#indent#Indent(lnum)
+
+    " First line has indent 0
+    if a:lnum == 1
+        return 0
+    endif
+
+    " If we can find an open parenthesis/bracket/brace, line up with it.
+    call cursor(a:lnum, 1)
+    let parlnum = s:SearchParensPair()
+    if parlnum > 0
+        let parcol = col('.')
+        let closing_paren = match(getline(a:lnum), '^\s*[])}]') != -1
+        if match(getline(parlnum), '[([{]\s*$', parcol - 1) != -1
+            if closing_paren
+                return indent(parlnum)
+            else
+                return indent(parlnum) + &shiftwidth
+            endif
+        else
+            return parcol
+        endif
+    endif
+
+    " Examine this line
+    let thisline = getline(a:lnum)
+    let thisindent = indent(a:lnum)
+
+    " If the line starts with 'elif' or 'else', line up with 'if' or 'elif'
+    if thisline =~ '^\s*\(elif\|else\)\>'
+        let bslnum = s:BlockStarter(a:lnum, '^\s*\(if\|elif\)\>')
+        if bslnum > 0
+            return indent(bslnum)
+        else
+            return -1
+        endif
+    endif
+
+    " If the line starts with 'except' or 'finally', line up with 'try'
+    " or 'except'
+    if thisline =~ '^\s*\(except\|finally\)\>'
+        let bslnum = s:BlockStarter(a:lnum, '^\s*\(try\|except\)\>')
+        if bslnum > 0
+            return indent(bslnum)
+        else
+            return -1
+        endif
+    endif
+
+    " Examine previous line
+    let plnum = a:lnum - 1
+    let pline = getline(plnum)
+    let sslnum = s:StatementStart(plnum)
+
+    " If the previous line is blank, keep the same indentation
+    if pline =~ '^\s*$'
+        return -1
+    endif
+
+    " If this line is explicitly joined, try to find an indentation that looks
+    " good.
+    if pline =~ '\\$'
+        let compound_statement = '^\s*\(if\|while\|for\s.*\sin\|except\)\s*'
+        let maybe_indent = matchend(getline(sslnum), compound_statement)
+        if maybe_indent != -1
+            return maybe_indent
+        else
+            return indent(sslnum) + &sw * 2
+        endif
+    endif
+
+    " If the previous line ended with a colon and is not a comment, indent
+    " relative to statement start.
+    if pline =~ ':\s*$' && pline !~ '^\s*#'
+        return indent(sslnum) + &sw
+    endif
+
+    " If the previous line was a stop-execution statement or a pass
+    if getline(sslnum) =~ '^\s*\(break\|continue\|raise\|return\|pass\)\>'
+        " See if the user has already dedented
+        if indent(a:lnum) > indent(sslnum) - &sw
+            " If not, recommend one dedent
+            return indent(sslnum) - &sw
+        endif
+        " Otherwise, trust the user
+        return -1
+    endif
+
+    " In all other cases, line up with the start of the previous statement.
+    return indent(sslnum)
+endfunction
+
+
+" Find backwards the closest open parenthesis/bracket/brace.
+function! s:SearchParensPair()
+    let line = line('.')
+    let col = col('.')
+
+    " Skip strings and comments and don't look too far
+    let skip = "line('.') < " . (line - 50) . " ? dummy :" .
+                \ 'synIDattr(synID(line("."), col("."), 0), "name") =~? ' .
+                \ '"string\\|comment"'
+
+    " Search for parentheses
+    call cursor(line, col)
+    let parlnum = searchpair('(', '', ')', 'bW', skip)
+    let parcol = col('.')
+
+    " Search for brackets
+    call cursor(line, col)
+    let par2lnum = searchpair('\[', '', '\]', 'bW', skip)
+    let par2col = col('.')
+
+    " Search for braces
+    call cursor(line, col)
+    let par3lnum = searchpair('{', '', '}', 'bW', skip)
+    let par3col = col('.')
+
+    " Get the closest match
+    if par2lnum > parlnum || (par2lnum == parlnum && par2col > parcol)
+        let parlnum = par2lnum
+        let parcol = par2col
+    endif
+    if par3lnum > parlnum || (par3lnum == parlnum && par3col > parcol)
+        let parlnum = par3lnum
+        let parcol = par3col
+    endif
+
+    " Put the cursor on the match
+    if parlnum > 0
+        call cursor(parlnum, parcol)
+    endif
+    return parlnum
+endfunction
+
+
+" Find the start of a multi-line statement
+function! s:StatementStart(lnum)
+    let lnum = a:lnum
+    while 1
+        if getline(lnum - 1) =~ '\\$'
+            let lnum = lnum - 1
+        else
+            call cursor(lnum, 1)
+            let maybe_lnum = s:SearchParensPair()
+            if maybe_lnum < 1
+                return lnum
+            else
+                let lnum = maybe_lnum
+            endif
+        endif
+    endwhile
+endfunction
+
+
+" Find the block starter that matches the current line
+function! s:BlockStarter(lnum, block_start_re)
+    let lnum = a:lnum
+    let maxindent = 10000       " whatever
+    while lnum > 1
+        let lnum = prevnonblank(lnum - 1)
+        if indent(lnum) < maxindent
+            if getline(lnum) =~ a:block_start_re
+                return lnum
+            else
+                let maxindent = indent(lnum)
+                " It's not worth going further if we reached the top level
+                if maxindent == 0
+                    return -1
+                endif
+            endif
+        endif
+    endwhile
+    return -1
+endfunction

autoload/pymode/lint.vim

@@ -0,0 +1,105 @@
+fun! pymode#lint#Check() "{{{
+    " DESC: Run checkers on current file.
+    "
+    if !g:pymode_lint | return | endif
+
+    if &modifiable && &modified
+        try
+            write
+        catch /E212/
+            echohl Error | echo "File modified and I can't save it. Cancel code checking." | echohl None
+            return 0
+        endtry
+    endif
+
+    let g:pymode_lint_buffer = bufnr('%')
+
+    py lint.check_file()
+
+endfunction " }}}
+
+
+fun! pymode#lint#Parse()
+    " DESC: Parse result of code checking.
+    "
+    call setqflist(g:qf_list, 'r')
+
+    if g:pymode_lint_signs
+        call pymode#PlaceSigns()
+    endif
+
+    if g:pymode_lint_cwindow
+        call pymode#QuickfixOpen(0, g:pymode_lint_hold, g:pymode_lint_maxheight, g:pymode_lint_minheight, g:pymode_lint_jump)
+    endif
+
+    if !len(g:qf_list)
+        call pymode#WideMessage('Code checking is completed. No errors found.')
+    endif
+
+endfunction
+
+
+fun! pymode#lint#Toggle() "{{{
+    let g:pymode_lint = g:pymode_lint ? 0 : 1
+    call pymode#lint#toggle_win(g:pymode_lint, "Pymode lint")
+endfunction "}}}
+
+
+fun! pymode#lint#ToggleWindow() "{{{
+    let g:pymode_lint_cwindow = g:pymode_lint_cwindow ? 0 : 1
+    call pymode#lint#toggle_win(g:pymode_lint_cwindow, "Pymode lint cwindow")
+endfunction "}}}
+
+
+fun! pymode#lint#ToggleChecker() "{{{
+    let g:pymode_lint_checker = g:pymode_lint_checker == "pylint" ? "pyflakes" : "pylint"
+    echomsg "Pymode lint checker: " . g:pymode_lint_checker
+endfunction "}}}
+
+
+fun! pymode#lint#toggle_win(toggle, msg) "{{{
+    if a:toggle
+        echomsg a:msg." enabled"
+        botright cwindow
+        if &buftype == "quickfix"
+            wincmd p
+        endif
+    else
+        echomsg a:msg." disabled"
+        cclose
+    endif
+endfunction "}}}
+
+
+fun! pymode#lint#show_errormessage() "{{{
+    if g:pymode_lint_buffer != bufnr('%')
+        return 0
+    endif
+    let errors = getqflist()
+    if !len(errors)
+        return
+    endif
+    let [_, line, _, _] = getpos(".")
+    for e in errors
+        if e['lnum'] == line
+            call pymode#WideMessage(e['text'])
+        else
+            echo
+        endif
+    endfor
+endfunction " }}}
+
+
+fun! pymode#lint#Auto() "{{{
+    if &modifiable && &modified
+        try
+            write
+        catch /E212/
+            echohl Error | echo "File modified and I can't save it. Cancel operation." | echohl None
+            return 0
+        endtry
+    endif
+    py auto.fix_current_file()
+    cclose
+    edit
+endfunction "}}}

autoload/pymode/motion.vim

@@ -0,0 +1,61 @@
+" Python-mode motion functions
+
+
+fun! pymode#motion#move(pattern, flags, ...) "{{{
+    let cnt = v:count1 - 1
+    let [line, column] = searchpos(a:pattern, a:flags . 'sW')
+    let indent = indent(line)
+    while cnt && line
+        let [line, column] = searchpos(a:pattern, a:flags . 'W')
+        if indent(line) == indent
+            let cnt = cnt - 1
+        endif
+    endwhile
+    return [line, column]
+endfunction "}}}
+
+
+fun! pymode#motion#vmove(pattern, flags) range "{{{
+    call cursor(a:lastline, 0)
+    let end = pymode#motion#move(a:pattern, a:flags)
+    call cursor(a:firstline, 0)
+    normal! v
+    call cursor(end)
+endfunction "}}} 
+
+
+fun! pymode#motion#pos_le(pos1, pos2) "{{{
+    return ((a:pos1[0] < a:pos2[0]) || (a:pos1[0] == a:pos2[0] && a:pos1[1] <= a:pos2[1]))
+endfunction "}}}
+
+
+fun! pymode#motion#select(pattern, inner) "{{{
+    let cnt = v:count1 - 1
+    let orig = getpos('.')[1:2]
+    let snum = pymode#BlockStart(orig[0], a:pattern)
+    if getline(snum) !~ a:pattern
+        return 0
+    endif
+    let enum = pymode#BlockEnd(snum, indent(snum))
+    while cnt
+        let lnum = search(a:pattern, 'nW')
+        if lnum
+            let enum = pymode#BlockEnd(lnum, indent(lnum))
+            call cursor(enum, 1)
+        endif
+        let cnt = cnt - 1
+    endwhile
+    if pymode#motion#pos_le([snum, 0], orig) && pymode#motion#pos_le(orig, [enum, 1])
+        if a:inner
+            let snum = snum + 1
+            let enum = prevnonblank(enum)
+        endif
+
+        call cursor(snum, 1)
+        normal! v
+        call cursor(enum, len(getline(enum)))
+    endif
+endfunction "}}}
+
+
+" vim: fdm=marker:fdl=0

autoload/pymode/queue.vim

@@ -0,0 +1,15 @@
+fun! pymode#queue#Poll() "{{{
+
+    " Check current tasks
+    py queue.check_task()
+
+    " Update interval
+    if mode() == 'i'
+        let p = getpos('.')
+        silent exe 'call feedkeys("\<Up>\<Down>", "n")'
+        call setpos('.', p)
+    else
+        call feedkeys("f\e", "n")
+    endif
+
+endfunction "}}}

autoload/pymode/run.vim

@@ -0,0 +1,20 @@
+" DESC: Save file if it modified and run python code
+fun! pymode#run#Run(line1, line2) "{{{
+    if &modifiable && &modified | write | endif	
+    py import StringIO
+    py sys.stdout, _ = StringIO.StringIO(), sys.stdout
+    call pymode#WideMessage("Code running.")
+    try
+        py execfile(vim.eval('expand("%s:p")'))
+        py sys.stdout, out = _, sys.stdout.getvalue()
+        call pymode#TempBuffer()
+        py vim.current.buffer.append(out.split('\n'), 0)
+        wincmd p
+        call pymode#WideMessage("")
+
+    catch /.*/
+
+        echohl Error | echo "Run-time error." | echohl none
+        
+    endtry
+endfunction "}}}

autoload/pymode/troubleshooting.vim

@@ -0,0 +1,87 @@
+" DESC: Get debug information about pymode problem
+fun! pymode#troubleshooting#Test() "{{{
+    new
+    setlocal buftype=nofile bufhidden=delete noswapfile nowrap
+
+    let os = "Unknown"
+    if has('win16') || has('win32') || has('win64')
+        let os = "Windows"
+    else
+        let os = substitute(system('uname'), "\n", "", "")
+    endif
+
+    call append('0', ['Pymode diagnostic',
+                  \ '===================',
+                  \ 'VIM:' . v:version . ', OS: ' . os .', multi_byte:' .  has('multi_byte') . ', pymode: ' . g:pymode_version,
+                  \ ''])
+
+    let python = 1
+    let output = []
+
+    if !exists('#filetypeplugin')
+        call append('$', ['WARNING: ', 'Python-mode required :filetype plugin indent on', ''])
+    endif
+
+    if !has('python')
+        call append('$', ['WARNING: ', 'Python-mode required vim compiled with +python.',
+                        \ '"lint, rope, run, doc, virtualenv" features disabled.', ''])
+        let python = 0
+    endif
+
+    call append('$', 'Pymode variables:')
+    call append('$', '-------------------')
+    call append('$', 'let pymode = ' . string(g:pymode))
+    if g:pymode
+        call append('$', 'let pymode_path = ' . string(g:pymode_path))
+        call append('$', 'let pymode_paths = ' . string(g:pymode_paths))
+
+        call append('$', 'let pymode_doc = ' . string(g:pymode_doc))
+        if g:pymode_doc
+            call append('$', 'let pymode_doc_key = ' . string(g:pymode_doc_key))
+        endif
+
+        call append('$', 'let pymode_run = ' . string(g:pymode_run))
+        if g:pymode_run
+            call append('$', 'let pymode_run_key = ' . string(g:pymode_run_key))
+        endif
+
+        call append('$', 'let pymode_lint = ' . string(g:pymode_lint))
+        if g:pymode_lint
+            call append('$', 'let pymode_lint_checker = ' . string(g:pymode_lint_checker))
+            call append('$', 'let pymode_lint_ignore = ' . string(g:pymode_lint_ignore))
+            call append('$', 'let pymode_lint_select = ' . string(g:pymode_lint_select))
+            call append('$', 'let pymode_lint_onfly = ' . string(g:pymode_lint_onfly))
+            call append('$', 'let pymode_lint_config = ' . string(g:pymode_lint_config))
+            call append('$', 'let pymode_lint_write = ' . string(g:pymode_lint_write))
+            call append('$', 'let pymode_lint_cwindow = ' . string(g:pymode_lint_cwindow))
+            call append('$', 'let pymode_lint_message = ' . string(g:pymode_lint_message))
+            call append('$', 'let pymode_lint_signs = ' . string(g:pymode_lint_signs))
+            call append('$', 'let pymode_lint_jump = ' . string(g:pymode_lint_jump))
+            call append('$', 'let pymode_lint_hold = ' . string(g:pymode_lint_hold))
+            call append('$', 'let pymode_lint_minheight = ' .  string(g:pymode_lint_minheight))
+            call append('$', 'let pymode_lint_maxheight = ' .  string(g:pymode_lint_maxheight)) 
+        endif
+
+        call append('$', 'let pymode_rope = ' . string(g:pymode_rope))
+        call append('$', 'let pymode_folding = ' . string(g:pymode_folding))
+        call append('$', 'let pymode_breakpoint = ' . string(g:pymode_breakpoint))
+        call append('$', 'let pymode_syntax = ' . string(g:pymode_syntax))
+        call append('$', 'let pymode_virtualenv = ' . string(g:pymode_virtualenv))
+        if g:pymode_virtualenv
+            call append('$', 'let pymode_virtualenv_enabled = ' .  string(g:pymode_virtualenv_enabled))
+        endif
+        call append('$', 'pymode_utils_whitespaces:' . string(g:pymode_utils_whitespaces))
+        call append('$', 'pymode_options:' . string(g:pymode_options))
+    endif
+
+    if python
+        call append('$', 'VIM python paths:')
+        call append('$', '-----------------')
+python << EOF
+vim.command('let l:output = %s' % repr(sys.path))
+EOF
+        call append('$', output)
+        call append('$', '')
+    endif
+    
+endfunction "}}}

autoload/pymode/virtualenv.vim

@@ -0,0 +1,28 @@
+fun! pymode#virtualenv#Activate() "{{{
+
+    if !exists("$VIRTUAL_ENV")
+        return
+    endif
+
+    for env in g:pymode_virtualenv_enabled
+        if env == $VIRTUAL_ENV
+            return 0
+        endif
+    endfor
+
+    call add(g:pymode_virtualenv_enabled, $VIRTUAL_ENV)
+
+python << EOF
+import sys, vim, os
+
+ve_dir = os.environ['VIRTUAL_ENV']
+ve_dir in sys.path or sys.path.insert(0, ve_dir)
+activate_this = os.path.join(os.path.join(ve_dir, 'bin'), 'activate_this.py')
+
+# Fix for windows
+if not os.path.exists(activate_this):
+    activate_this = os.path.join(os.path.join(ve_dir, 'Scripts'), 'activate_this.py')
+
+execfile(activate_this, dict(__file__=activate_this))
+EOF
+endfunction "}}}

autoload/snipMate.vim

@@ -0,0 +1,435 @@
+fun! Filename(...)
+	let filename = expand('%:t:r')
+	if filename == '' | return a:0 == 2 ? a:2 : '' | endif
+	return !a:0 || a:1 == '' ? filename : substitute(a:1, '$1', filename, 'g')
+endf
+
+fun s:RemoveSnippet()
+	unl! g:snipPos s:curPos s:snipLen s:endCol s:endLine s:prevLen
+	     \ s:lastBuf s:oldWord
+	if exists('s:update')
+		unl s:startCol s:origWordLen s:update
+		if exists('s:oldVars') | unl s:oldVars s:oldEndCol | endif
+	endif
+	aug! snipMateAutocmds
+endf
+
+fun snipMate#expandSnip(snip, col)
+	let lnum = line('.') | let col = a:col
+
+	let snippet = s:ProcessSnippet(a:snip)
+	" Avoid error if eval evaluates to nothing
+	if snippet == '' | return '' | endif
+
+	" Expand snippet onto current position with the tab stops removed
+	let snipLines = split(substitute(snippet, '$\d\+\|${\d\+.\{-}}', '', 'g'), "\n", 1)
+
+	let line = getline(lnum)
+	let afterCursor = strpart(line, col - 1)
+	" Keep text after the cursor
+	if afterCursor != "\t" && afterCursor != ' '
+		let line = strpart(line, 0, col - 1)
+		let snipLines[-1] .= afterCursor
+	else
+		let afterCursor = ''
+		" For some reason the cursor needs to move one right after this
+		if line != '' && col == 1 && &ve != 'all' && &ve != 'onemore'
+			let col += 1
+		endif
+	endif
+
+	call setline(lnum, line.snipLines[0])
+
+	" Autoindent snippet according to previous indentation
+	let indent = matchend(line, '^.\{-}\ze\(\S\|$\)') + 1
+	call append(lnum, map(snipLines[1:], "'".strpart(line, 0, indent - 1)."'.v:val"))
+
+	" Open any folds snippet expands into
+	if &fen | sil! exe lnum.','.(lnum + len(snipLines) - 1).'foldopen' | endif
+
+	let [g:snipPos, s:snipLen] = s:BuildTabStops(snippet, lnum, col - indent, indent)
+
+	if s:snipLen
+		aug snipMateAutocmds
+			au CursorMovedI * call s:UpdateChangedSnip(0)
+			au InsertEnter * call s:UpdateChangedSnip(1)
+		aug END
+		let s:lastBuf = bufnr(0) " Only expand snippet while in current buffer
+		let s:curPos = 0
+		let s:endCol = g:snipPos[s:curPos][1]
+		let s:endLine = g:snipPos[s:curPos][0]
+
+		call cursor(g:snipPos[s:curPos][0], g:snipPos[s:curPos][1])
+		let s:prevLen = [line('$'), col('$')]
+		if g:snipPos[s:curPos][2] != -1 | return s:SelectWord() | endif
+	else
+		unl g:snipPos s:snipLen
+		" Place cursor at end of snippet if no tab stop is given
+		let newlines = len(snipLines) - 1
+		call cursor(lnum + newlines, indent + len(snipLines[-1]) - len(afterCursor)
+					\ + (newlines ? 0: col - 1))
+	endif
+	return ''
+endf
+
+" Prepare snippet to be processed by s:BuildTabStops
+fun s:ProcessSnippet(snip)
+	let snippet = a:snip
+	" Evaluate eval (`...`) expressions.
+	" Backquotes prefixed with a backslash "\" are ignored.
+	" Using a loop here instead of a regex fixes a bug with nested "\=".
+	if stridx(snippet, '`') != -1
+		while match(snippet, '\(^\|[^\\]\)`.\{-}[^\\]`') != -1
+			let snippet = substitute(snippet, '\(^\|[^\\]\)\zs`.\{-}[^\\]`\ze',
+		                \ substitute(eval(matchstr(snippet, '\(^\|[^\\]\)`\zs.\{-}[^\\]\ze`')),
+		                \ "\n\\%$", '', ''), '')
+		endw
+		let snippet = substitute(snippet, "\r", "\n", 'g')
+		let snippet = substitute(snippet, '\\`', '`', 'g')
+	endif
+
+	" Place all text after a colon in a tab stop after the tab stop
+	" (e.g. "${#:foo}" becomes "${:foo}foo").
+	" This helps tell the position of the tab stops later.
+	let snippet = substitute(snippet, '${\d\+:\(.\{-}\)}', '&\1', 'g')
+
+	" Update the a:snip so that all the $# become the text after
+	" the colon in their associated ${#}.
+	" (e.g. "${1:foo}" turns all "$1"'s into "foo")
+	let i = 1
+	while stridx(snippet, '${'.i) != -1
+		let s = matchstr(snippet, '${'.i.':\zs.\{-}\ze}')
+		if s != ''
+			let snippet = substitute(snippet, '$'.i, s.'&', 'g')
+		endif
+		let i += 1
+	endw
+
+	if &et " Expand tabs to spaces if 'expandtab' is set.
+		return substitute(snippet, '\t', repeat(' ', &sts ? &sts : &sw), 'g')
+	endif
+	return snippet
+endf
+
+" Counts occurences of haystack in needle
+fun s:Count(haystack, needle)
+	let counter = 0
+	let index = stridx(a:haystack, a:needle)
+	while index != -1
+		let index = stridx(a:haystack, a:needle, index+1)
+		let counter += 1
+	endw
+	return counter
+endf
+
+" Builds a list of a list of each tab stop in the snippet containing:
+" 1.) The tab stop's line number.
+" 2.) The tab stop's column number
+"     (by getting the length of the string between the last "\n" and the
+"     tab stop).
+" 3.) The length of the text after the colon for the current tab stop
+"     (e.g. "${1:foo}" would return 3). If there is no text, -1 is returned.
+" 4.) If the "${#:}" construct is given, another list containing all
+"     the matches of "$#", to be replaced with the placeholder. This list is
+"     composed the same way as the parent; the first item is the line number,
+"     and the second is the column.
+fun s:BuildTabStops(snip, lnum, col, indent)
+	let snipPos = []
+	let i = 1
+	let withoutVars = substitute(a:snip, '$\d\+', '', 'g')
+	while stridx(a:snip, '${'.i) != -1
+		let beforeTabStop = matchstr(withoutVars, '^.*\ze${'.i.'\D')
+		let withoutOthers = substitute(withoutVars, '${\('.i.'\D\)\@!\d\+.\{-}}', '', 'g')
+
+		let j = i - 1
+		call add(snipPos, [0, 0, -1])
+		let snipPos[j][0] = a:lnum + s:Count(beforeTabStop, "\n")
+		let snipPos[j][1] = a:indent + len(matchstr(withoutOthers, '.*\(\n\|^\)\zs.*\ze${'.i.'\D'))
+		if snipPos[j][0] == a:lnum | let snipPos[j][1] += a:col | endif
+
+		" Get all $# matches in another list, if ${#:name} is given
+		if stridx(withoutVars, '${'.i.':') != -1
+			let snipPos[j][2] = len(matchstr(withoutVars, '${'.i.':\zs.\{-}\ze}'))
+			let dots = repeat('.', snipPos[j][2])
+			call add(snipPos[j], [])
+			let withoutOthers = substitute(a:snip, '${\d\+.\{-}}\|$'.i.'\@!\d\+', '', 'g')
+			while match(withoutOthers, '$'.i.'\(\D\|$\)') != -1
+				let beforeMark = matchstr(withoutOthers, '^.\{-}\ze'.dots.'$'.i.'\(\D\|$\)')
+				call add(snipPos[j][3], [0, 0])
+				let snipPos[j][3][-1][0] = a:lnum + s:Count(beforeMark, "\n")
+				let snipPos[j][3][-1][1] = a:indent + (snipPos[j][3][-1][0] > a:lnum
+				                           \ ? len(matchstr(beforeMark, '.*\n\zs.*'))
+				                           \ : a:col + len(beforeMark))
+				let withoutOthers = substitute(withoutOthers, '$'.i.'\ze\(\D\|$\)', '', '')
+			endw
+		endif
+		let i += 1
+	endw
+	return [snipPos, i - 1]
+endf
+
+fun snipMate#jumpTabStop(backwards)
+	let leftPlaceholder = exists('s:origWordLen')
+	                      \ && s:origWordLen != g:snipPos[s:curPos][2]
+	if leftPlaceholder && exists('s:oldEndCol')
+		let startPlaceholder = s:oldEndCol + 1
+	endif
+
+	if exists('s:update')
+		call s:UpdatePlaceholderTabStops()
+	else
+		call s:UpdateTabStops()
+	endif
+
+	" Don't reselect placeholder if it has been modified
+	if leftPlaceholder && g:snipPos[s:curPos][2] != -1
+		if exists('startPlaceholder')
+			let g:snipPos[s:curPos][1] = startPlaceholder
+		else
+			let g:snipPos[s:curPos][1] = col('.')
+			let g:snipPos[s:curPos][2] = 0
+		endif
+	endif
+
+	let s:curPos += a:backwards ? -1 : 1
+	" Loop over the snippet when going backwards from the beginning
+	if s:curPos < 0 | let s:curPos = s:snipLen - 1 | endif
+
+	if s:curPos == s:snipLen
+		let sMode = s:endCol == g:snipPos[s:curPos-1][1]+g:snipPos[s:curPos-1][2]
+		call s:RemoveSnippet()
+		return sMode ? "\<tab>" : TriggerSnippet()
+	endif
+
+	call cursor(g:snipPos[s:curPos][0], g:snipPos[s:curPos][1])
+
+	let s:endLine = g:snipPos[s:curPos][0]
+	let s:endCol = g:snipPos[s:curPos][1]
+	let s:prevLen = [line('$'), col('$')]
+
+	return g:snipPos[s:curPos][2] == -1 ? '' : s:SelectWord()
+endf
+
+fun s:UpdatePlaceholderTabStops()
+	let changeLen = s:origWordLen - g:snipPos[s:curPos][2]
+	unl s:startCol s:origWordLen s:update
+	if !exists('s:oldVars') | return | endif
+	" Update tab stops in snippet if text has been added via "$#"
+	" (e.g., in "${1:foo}bar$1${2}").
+	if changeLen != 0
+		let curLine = line('.')
+
+		for pos in g:snipPos
+			if pos == g:snipPos[s:curPos] | continue | endif
+			let changed = pos[0] == curLine && pos[1] > s:oldEndCol
+			let changedVars = 0
+			let endPlaceholder = pos[2] - 1 + pos[1]
+			" Subtract changeLen from each tab stop that was after any of
+			" the current tab stop's placeholders.
+			for [lnum, col] in s:oldVars
+				if lnum > pos[0] | break | endif
+				if pos[0] == lnum
+					if pos[1] > col || (pos[2] == -1 && pos[1] == col)
+						let changed += 1
+					elseif col < endPlaceholder
+						let changedVars += 1
+					endif
+				endif
+			endfor
+			let pos[1] -= changeLen * changed
+			let pos[2] -= changeLen * changedVars " Parse variables within placeholders
+                                                  " e.g., "${1:foo} ${2:$1bar}"
+
+			if pos[2] == -1 | continue | endif
+			" Do the same to any placeholders in the other tab stops.
+			for nPos in pos[3]
+				let changed = nPos[0] == curLine && nPos[1] > s:oldEndCol
+				for [lnum, col] in s:oldVars
+					if lnum > nPos[0] | break | endif
+					if nPos[0] == lnum && nPos[1] > col
+						let changed += 1
+					endif
+				endfor
+				let nPos[1] -= changeLen * changed
+			endfor
+		endfor
+	endif
+	unl s:endCol s:oldVars s:oldEndCol
+endf
+
+fun s:UpdateTabStops()
+	let changeLine = s:endLine - g:snipPos[s:curPos][0]
+	let changeCol = s:endCol - g:snipPos[s:curPos][1]
+	if exists('s:origWordLen')
+		let changeCol -= s:origWordLen
+		unl s:origWordLen
+	endif
+	let lnum = g:snipPos[s:curPos][0]
+	let col = g:snipPos[s:curPos][1]
+	" Update the line number of all proceeding tab stops if <cr> has
+	" been inserted.
+	if changeLine != 0
+		let changeLine -= 1
+		for pos in g:snipPos
+			if pos[0] >= lnum
+				if pos[0] == lnum | let pos[1] += changeCol | endif
+				let pos[0] += changeLine
+			endif
+			if pos[2] == -1 | continue | endif
+			for nPos in pos[3]
+				if nPos[0] >= lnum
+					if nPos[0] == lnum | let nPos[1] += changeCol | endif
+					let nPos[0] += changeLine
+				endif
+			endfor
+		endfor
+	elseif changeCol != 0
+		" Update the column of all proceeding tab stops if text has
+		" been inserted/deleted in the current line.
+		for pos in g:snipPos
+			if pos[1] >= col && pos[0] == lnum
+				let pos[1] += changeCol
+			endif
+			if pos[2] == -1 | continue | endif
+			for nPos in pos[3]
+				if nPos[0] > lnum | break | endif
+				if nPos[0] == lnum && nPos[1] >= col
+					let nPos[1] += changeCol
+				endif
+			endfor
+		endfor
+	endif
+endf
+
+fun s:SelectWord()
+	let s:origWordLen = g:snipPos[s:curPos][2]
+	let s:oldWord = strpart(getline('.'), g:snipPos[s:curPos][1] - 1,
+				\ s:origWordLen)
+	let s:prevLen[1] -= s:origWordLen
+	if !empty(g:snipPos[s:curPos][3])
+		let s:update = 1
+		let s:endCol = -1
+		let s:startCol = g:snipPos[s:curPos][1] - 1
+	endif
+	if !s:origWordLen | return '' | endif
+	let l = col('.') != 1 ? 'l' : ''
+	if &sel == 'exclusive'
+		return "\<esc>".l.'v'.s:origWordLen."l\<c-g>"
+	endif
+	return s:origWordLen == 1 ? "\<esc>".l.'gh'
+							\ : "\<esc>".l.'v'.(s:origWordLen - 1)."l\<c-g>"
+endf
+
+" This updates the snippet as you type when text needs to be inserted
+" into multiple places (e.g. in "${1:default text}foo$1bar$1",
+" "default text" would be highlighted, and if the user types something,
+" UpdateChangedSnip() would be called so that the text after "foo" & "bar"
+" are updated accordingly)
+"
+" It also automatically quits the snippet if the cursor is moved out of it
+" while in insert mode.
+fun s:UpdateChangedSnip(entering)
+	if exists('g:snipPos') && bufnr(0) != s:lastBuf
+		call s:RemoveSnippet()
+	elseif exists('s:update') " If modifying a placeholder
+		if !exists('s:oldVars') && s:curPos + 1 < s:snipLen
+			" Save the old snippet & word length before it's updated
+			" s:startCol must be saved too, in case text is added
+			" before the snippet (e.g. in "foo$1${2}bar${1:foo}").
+			let s:oldEndCol = s:startCol
+			let s:oldVars = deepcopy(g:snipPos[s:curPos][3])
+		endif
+		let col = col('.') - 1
+
+		if s:endCol != -1
+			let changeLen = col('$') - s:prevLen[1]
+			let s:endCol += changeLen
+		else " When being updated the first time, after leaving select mode
+			if a:entering | return | endif
+			let s:endCol = col - 1
+		endif
+
+		" If the cursor moves outside the snippet, quit it
+		if line('.') != g:snipPos[s:curPos][0] || col < s:startCol ||
+					\ col - 1 > s:endCol
+			unl! s:startCol s:origWordLen s:oldVars s:update
+			return s:RemoveSnippet()
+		endif
+
+		call s:UpdateVars()
+		let s:prevLen[1] = col('$')
+	elseif exists('g:snipPos')
+		if !a:entering && g:snipPos[s:curPos][2] != -1
+			let g:snipPos[s:curPos][2] = -2
+		endif
+
+		let col = col('.')
+		let lnum = line('.')
+		let changeLine = line('$') - s:prevLen[0]
+
+		if lnum == s:endLine
+			let s:endCol += col('$') - s:prevLen[1]
+			let s:prevLen = [line('$'), col('$')]
+		endif
+		if changeLine != 0
+			let s:endLine += changeLine
+			let s:endCol = col
+		endif
+
+		" Delete snippet if cursor moves out of it in insert mode
+		if (lnum == s:endLine && (col > s:endCol || col < g:snipPos[s:curPos][1]))
+			\ || lnum > s:endLine || lnum < g:snipPos[s:curPos][0]
+			call s:RemoveSnippet()
+		endif
+	endif
+endf
+
+" This updates the variables in a snippet when a placeholder has been edited.
+" (e.g., each "$1" in "${1:foo} $1bar $1bar")
+fun s:UpdateVars()
+	let newWordLen = s:endCol - s:startCol + 1
+	let newWord = strpart(getline('.'), s:startCol, newWordLen)
+	if newWord == s:oldWord || empty(g:snipPos[s:curPos][3])
+		return
+	endif
+
+	let changeLen = g:snipPos[s:curPos][2] - newWordLen
+	let curLine = line('.')
+	let startCol = col('.')
+	let oldStartSnip = s:startCol
+	let updateTabStops = changeLen != 0
+	let i = 0
+
+	for [lnum, col] in g:snipPos[s:curPos][3]
+		if updateTabStops
+			let start = s:startCol
+			if lnum == curLine && col <= start
+				let s:startCol -= changeLen
+				let s:endCol -= changeLen
+			endif
+			for nPos in g:snipPos[s:curPos][3][(i):]
+				" This list is in ascending order, so quit if we've gone too far.
+				if nPos[0] > lnum | break | endif
+				if nPos[0] == lnum && nPos[1] > col
+					let nPos[1] -= changeLen
+				endif
+			endfor
+			if lnum == curLine && col > start
+				let col -= changeLen
+				let g:snipPos[s:curPos][3][i][1] = col
+			endif
+			let i += 1
+		endif
+
+		" "Very nomagic" is used here to allow special characters.
+		call setline(lnum, substitute(getline(lnum), '\%'.col.'c\V'.
+						\ escape(s:oldWord, '\'), escape(newWord, '\&'), ''))
+	endfor
+	if oldStartSnip != s:startCol
+		call cursor(0, startCol + s:startCol - oldStartSnip)
+	endif
+
+	let s:oldWord = newWord
+	let g:snipPos[s:curPos][2] = newWordLen
+endf
+" vim:noet:sw=4:ts=4:ft=vim

colors/vividchalk.vim

@@ -0,0 +1,192 @@
+" Vim color scheme
+" Name:         vividchalk.vim
+" Author:       Tim Pope <vimNOSPAM@tpope.info>
+" Version:      2.0
+" GetLatestVimScripts: 1891 1 :AutoInstall: vividchalk.vim
+
+" Based on the Vibrank Ink theme for TextMate
+" Distributable under the same terms as Vim itself (see :help license)
+
+if has("gui_running")
+    set background=dark
+endif
+hi clear
+if exists("syntax_on")
+   syntax reset
+endif
+
+let colors_name = "vividchalk"
+
+" First two functions adapted from inkpot.vim
+
+" map a urxvt cube number to an xterm-256 cube number
+fun! s:M(a)
+    return strpart("0245", a:a, 1) + 0
+endfun
+
+" map a urxvt colour to an xterm-256 colour
+fun! s:X(a)
+    if &t_Co == 88
+        return a:a
+    else
+        if a:a == 8
+            return 237
+        elseif a:a < 16
+            return a:a
+        elseif a:a > 79
+            return 232 + (3 * (a:a - 80))
+        else
+            let l:b = a:a - 16
+            let l:x = l:b % 4
+            let l:y = (l:b / 4) % 4
+            let l:z = (l:b / 16)
+            return 16 + s:M(l:x) + (6 * s:M(l:y)) + (36 * s:M(l:z))
+        endif
+    endif
+endfun
+
+function! E2T(a)
+    return s:X(a:a)
+endfunction
+
+function! s:choose(mediocre,good)
+    if &t_Co != 88 && &t_Co != 256
+        return a:mediocre
+    else
+        return s:X(a:good)
+    endif
+endfunction
+
+function! s:hifg(group,guifg,first,second,...)
+    if a:0 && &t_Co == 256
+        let ctermfg = a:1
+    else
+        let ctermfg = s:choose(a:first,a:second)
+    endif
+    exe "highlight ".a:group." guifg=".a:guifg." ctermfg=".ctermfg
+endfunction
+
+function! s:hibg(group,guibg,first,second)
+    let ctermbg = s:choose(a:first,a:second)
+    exe "highlight ".a:group." guibg=".a:guibg." ctermbg=".ctermbg
+endfunction
+
+hi link railsMethod         PreProc
+hi link rubyDefine          Keyword
+hi link rubySymbol          Constant
+hi link rubyAccess          rubyMethod
+hi link rubyAttribute       rubyMethod
+hi link rubyEval            rubyMethod
+hi link rubyException       rubyMethod
+hi link rubyInclude         rubyMethod
+hi link rubyStringDelimiter rubyString
+hi link rubyRegexp          Regexp
+hi link rubyRegexpDelimiter rubyRegexp
+"hi link rubyConstant        Variable
+"hi link rubyGlobalVariable  Variable
+"hi link rubyClassVariable   Variable
+"hi link rubyInstanceVariable Variable
+hi link javascriptRegexpString  Regexp
+hi link javascriptNumber        Number
+hi link javascriptNull          Constant
+highlight link diffAdded        String
+highlight link diffRemoved      Statement
+highlight link diffLine         PreProc
+highlight link diffSubname      Comment
+
+call s:hifg("Normal","#EEEEEE","White",87)
+if &background == "light" || has("gui_running")
+    hi Normal guibg=Black ctermbg=Black
+else
+    hi Normal guibg=Black ctermbg=NONE
+endif
+highlight StatusLine    guifg=Black   guibg=#aabbee gui=bold ctermfg=Black ctermbg=White  cterm=bold
+highlight StatusLineNC  guifg=#444444 guibg=#aaaaaa gui=none ctermfg=Black ctermbg=Grey   cterm=none
+"if &t_Co == 256
+    "highlight StatusLine ctermbg=117
+"else
+    "highlight StatusLine ctermbg=43
+"endif
+
+highlight Ignore        ctermfg=Black
+highlight WildMenu      guifg=Black   guibg=#ffff00 gui=bold ctermfg=Black ctermbg=Yellow cterm=bold
+highlight Cursor        guifg=Black guibg=White ctermfg=Black ctermbg=White
+call s:hibg("ColorColumn","#333333","DarkGrey",81)
+call s:hibg("CursorLine","#333333","DarkGrey",81)
+call s:hibg("CursorColumn","#333333","DarkGrey",81)
+highlight NonText       guifg=#404040 ctermfg=8
+highlight SpecialKey    guifg=#404040 ctermfg=8
+highlight Directory     none
+high link Directory     Identifier
+highlight ErrorMsg      guibg=Red ctermbg=DarkRed guifg=NONE ctermfg=NONE
+highlight Search        guifg=NONE ctermfg=NONE gui=none cterm=none
+call s:hibg("Search"    ,"#555555","DarkBlue",81)
+highlight IncSearch     guifg=White guibg=Black ctermfg=White ctermbg=Black
+highlight MoreMsg       guifg=#00AA00 ctermfg=Green
+highlight LineNr        guifg=#DDEEFF ctermfg=White
+call s:hibg("LineNr"    ,"#222222","DarkBlue",80)
+highlight Question      none
+high link Question      MoreMsg
+highlight Title         guifg=Magenta ctermfg=Magenta
+highlight VisualNOS     gui=none cterm=none
+call s:hibg("Visual"    ,"#555577","LightBlue",83)
+call s:hibg("VisualNOS" ,"#444444","DarkBlue",81)
+call s:hibg("MatchParen","#1100AA","DarkBlue",18)
+highlight WarningMsg    guifg=Red ctermfg=Red
+highlight Error         ctermbg=DarkRed
+highlight SpellBad      ctermbg=DarkRed
+" FIXME: Comments
+highlight SpellRare     ctermbg=DarkMagenta
+highlight SpellCap      ctermbg=DarkBlue
+highlight SpellLocal    ctermbg=DarkCyan
+
+call s:hibg("Folded"    ,"#110077","DarkBlue",17)
+call s:hifg("Folded"    ,"#aaddee","LightCyan",63)
+highlight FoldColumn    none
+high link FoldColumn    Folded
+highlight DiffAdd       ctermbg=4 guibg=DarkBlue
+highlight DiffChange    ctermbg=5 guibg=DarkMagenta
+highlight DiffDelete    ctermfg=12 ctermbg=6 gui=bold guifg=Blue guibg=DarkCyan
+highlight DiffText      ctermbg=DarkRed
+highlight DiffText      cterm=bold ctermbg=9 gui=bold guibg=Red
+
+highlight Pmenu         guifg=White ctermfg=White gui=bold cterm=bold
+highlight PmenuSel      guifg=White ctermfg=White gui=bold cterm=bold
+call s:hibg("Pmenu"     ,"#000099","Blue",18)
+call s:hibg("PmenuSel"  ,"#5555ff","DarkCyan",39)
+highlight PmenuSbar     guibg=Grey ctermbg=Grey
+highlight PmenuThumb    guibg=White ctermbg=White
+highlight TabLine       gui=underline cterm=underline
+call s:hifg("TabLine"   ,"#bbbbbb","LightGrey",85)
+call s:hibg("TabLine"   ,"#333333","DarkGrey",80)
+highlight TabLineSel    guifg=White guibg=Black ctermfg=White ctermbg=Black
+highlight TabLineFill   gui=underline cterm=underline
+call s:hifg("TabLineFill","#bbbbbb","LightGrey",85)
+call s:hibg("TabLineFill","#808080","Grey",83)
+
+hi Type gui=none
+hi Statement gui=none
+if !has("gui_mac")
+    " Mac GUI degrades italics to ugly underlining.
+    hi Comment gui=italic
+    hi railsUserClass  gui=italic
+    hi railsUserMethod gui=italic
+endif
+hi Identifier cterm=none
+" Commented numbers at the end are *old* 256 color values
+"highlight PreProc       guifg=#EDF8F9
+call s:hifg("Comment"        ,"#9933CC","DarkMagenta",34) " 92
+" 26 instead?
+call s:hifg("Constant"       ,"#339999","DarkCyan",21) " 30
+call s:hifg("rubyNumber"     ,"#CCFF33","Yellow",60) " 190
+call s:hifg("String"         ,"#66FF00","LightGreen",44,82) " 82
+call s:hifg("Identifier"     ,"#FFCC00","Yellow",72) " 220
+call s:hifg("Statement"      ,"#FF6600","Brown",68) " 202
+call s:hifg("PreProc"        ,"#AAFFFF","LightCyan",47) " 213
+call s:hifg("railsUserMethod","#AACCFF","LightCyan",27)
+call s:hifg("Type"           ,"#AAAA77","Grey",57) " 101
+call s:hifg("railsUserClass" ,"#AAAAAA","Grey",7) " 101
+call s:hifg("Special"        ,"#33AA00","DarkGreen",24) " 7
+call s:hifg("Regexp"         ,"#44B4CC","DarkCyan",21) " 74
+call s:hifg("rubyMethod"     ,"#DDE93D","Yellow",77) " 191
+"highlight railsMethod   guifg=#EE1122 ctermfg=1

doc/conque_term.txt

@@ -0,0 +1 @@
+/usr/share/vim-conque/doc/conque_term.txt

doc/pymode.txt

@@ -0,0 +1,549 @@
+*pymode.txt*  *python-mode.txt*  Python-mode for vim!
+
+    ____  _  _  ____  _   _  _____  _  _     __  __  _____  ____  ____      ~
+   (  _ \( \/ )(_  _)( )_( )(  _  )( \( )___(  \/  )(  _  )(  _ \( ___)     ~
+    )___/ \  /   )(   ) _ (  )(_)(  )  ((___))    (  )(_)(  )(_) ))__)      ~
+   (__)   (__)  (__) (_) (_)(_____)(_)\_)   (_/\/\_)(_____)(____/(____)     ~
+
+
+                          Version: 0.6.9
+
+==============================================================================
+CONTENTS                                                  *Python-mode-contents*
+
+    1.Intro...................................|PythonMode|
+    2.Options.................................|PythonModeOptions|
+        2.1.Customisation details.............|PythonModeOptionsDetails|
+        2.2.Modeline..........................|PythonModeModeline|
+    3.Default Keys............................|PythonModeKeys|
+    4.Commands................................|PythonModeCommands|
+    5.FAQ.....................................|PythonModeFAQ|
+    6.Credits.................................|PythonModeCredits|
+    7.License.................................|PythonModeLicense|
+
+==============================================================================
+1. Intro ~
+                                                                    *PythonMode*
+
+Python-mode is a vim plugin that allows you to use the pylint, rope, and pydoc
+libraries in vim to provide features like python code bug checking,
+refactoring, and some other useful things.
+
+This plugin allow you create python code in vim very easily. There is no need
+to install the pylint or rope libraries on your system.
+
+
+==============================================================================
+2. Options ~
+                                                             *PythonModeOptions*
+
+	Note:
+        Pylint options (ex. disable messages) may be defined in '$HOME/pylint.rc'
+        See pylint documentation.
+
+This script provides the following options that can customise the behaviour of
+PythonMode. These options should be set in your vimrc.
+
+|'pymode_paths'|                    Additional python paths for pymode
+
+|'pymode_doc'|                      Turns off the documentation script
+
+|'pymode_doc_key'|                  Key for show documentation
+
+|'pymode_run'|                      Turns off the run code script
+
+|'pymode_run_key'|                  Key for run python code
+
+|'pymode_lint'|                     Turns off pylint script
+
+|'pymode_lint_checker'|             Switch code checkers (pylint, pyflakes, pep8, mccabe)
+
+|'pymode_lint_ignore'|              Skip errors and warnings
+
+|'pymode_lint_select'|              Select errors and warnings
+
+|'pymode_lint_onfly'|               Run linter on the fly
+
+|'pymode_lint_config'|              Filepath to pylint configuration
+
+|'pymode_lint_write'|               Check code every save
+
+|'pymode_lint_cwindow'|             Show cwindow
+
+|'pymode_lint_message'|             Show current line errors in bottom
+
+|'pymode_lint_signs'|               Place signs
+
+|'pymode_lint_jump'|                Auto jump to first error
+
+|'pymode_lint_hold'|                Hold cursor in current window
+
+|'pymode_lint_minheight'|           Minimal height of pylint error window
+
+|'pymode_lint_mccabe_complexity'|   Maximum allowed mccabe complexity
+
+|'pymode_lint_maxheight'|           Maximal height of pylint error window
+
+|'pymode_rope'|                     Turns off rope script
+
+|'pymode_folding'|                  Turns on/off python folding
+
+|'pymode_breakpoint'|               Turns off breakpoint script
+
+|'pymode_breakpoint_key'|           Key for breakpoint
+
+|'pymode_virtualenv'|               Turns off virtualenv
+
+|'pymode_utils_whitespaces'|        Remove unused whitespaces
+
+|'pymode_syntax'|                   Turns off the custom syntax highlighting
+
+|'pymode_indent'|                   Enable/Disable pymode PEP8 indentation
+
+|'pymode_options'|                  Set default pymode options for
+                                  python codding
+
+|'pymode_motion'|                   Enable pymode motion stuff
+
+        Note:
+        Also see |ropevim.txt|
+
+
+------------------------------------------------------------------------------
+2.1. Customisation details ~
+                                                      *PythonModeOptionsDetails*
+
+To enable any of the options below you should put the given line in your
+'$HOME/.vimrc'. See |vimrc-intro|.
+
+------------------------------------------------------------------------------
+2.2. Modeline ~
+                                                      *PythonModeModeline*
+
+Feature like VIM modeline `:help modeline`. Allow changing pymode options for
+edited file. Pymode modeline should always be the last line in the file and
+look like:
+
+>
+    # pymode:lint_ignore=E0202:doc=0:lint_write=0
+<
+
+Examples:
+
+Disable folding on current file:
+>
+    # pymode:folding=0
+<
+
+Set linters and mccabe complexity.
+>
+    # pymode:lint_checker=pip,mccabe:lint_mccabe_complexity=10
+<
+
+This changes will work only in current buffer.
+
+------------------------------------------------------------------------------
+                                                                *'pymode_paths'*
+Values: List of strings
+Default: [].
+
+This option set additional python import paths
+
+------------------------------------------------------------------------------
+                                                                  *'pymode_doc'*
+Values: 0 or 1.
+Default: 1.
+
+If this option is set to 0 then the doc script is disabled.
+
+------------------------------------------------------------------------------
+                                                              *'pymode_doc_key'*
+Default: 'K'.
+
+Set key for show python documentation.
+
+------------------------------------------------------------------------------
+                                                                  *'pymode_run'*
+Values: 0 or 1.
+Default: 1.
+
+If this option is set to 0 then run script is disabled.
+
+------------------------------------------------------------------------------
+                                                              *'pymode_run_key'*
+Default: '<leader>r'.
+
+Set key for run python code.
+
+------------------------------------------------------------------------------
+                                                                 *'pymode_lint'*
+Values: 0 or 1.
+Default: 1.
+
+If this option is set to 0 then the pylint script is disabled.
+
+------------------------------------------------------------------------------
+                                                         *'pymode_lint_checker'*
+Values: "pylint", "pyflakes", "pep8", "mccabe"
+        You can set many checkers. E.g. "pyflakes,pep8,mccabe" ~
+
+Default: "pyflakes,pep8,mccabe".
+
+This option sets code checkers.
+
+------------------------------------------------------------------------------
+                                                          *'pymode_lint_ignore'*
+Values: IDs of errors, separated by commas or empty strings
+        E.g. "E501,W002", "E2,W" (Skip all Warnings and Errors startswith E2) and etc ~
+
+Default: "E501".
+
+Skip errors and warnings.
+See also: |'pymode_lint_select'|, |'pymode_lint_config'|
+
+------------------------------------------------------------------------------
+                                                         *'pymode_lint_select'*
+Values: IDs of errors, separated by commas or empty strings
+        E.g. "W002,C" Force W002 and all C-ids ~
+
+Default: "".
+
+Select errors and warnings.
+See also: |'pymode_lint_ignore'|, |'pymode_lint_config'|
+
+------------------------------------------------------------------------------
+                                                           *'pymode_lint_onfly'*
+Values: 0 or 1
+Default: 0
+
+This option enables "on the fly" code checking
+
+------------------------------------------------------------------------------
+                                                          *'pymode_lint_config'*
+Values: 'Path to pylint configuration file'
+Default: "$HOME/.pylintrc"
+
+This option sets the path to the pylint configuration file. If the
+file is not found, use the 'pylintrc' file from python-mode sources.
+
+See also: |'pymode_lint_ignore'|, |'pymode_lint_select'|
+
+------------------------------------------------------------------------------
+                                                           *'pymode_lint_write'*
+Values: 0 or 1.
+Default: 1.
+
+If this option is set to 0, then pylint auto-checking on every save is
+disabled.
+
+------------------------------------------------------------------------------
+                                                         *'pymode_lint_cwindow'*
+Values: 0 or 1.
+Default: 1.
+
+If this option is set to 0 then pylint will not show cwindow.
+
+------------------------------------------------------------------------------
+                                                         *'pymode_lint_message'*
+Values: 0 or 1.
+Default: 1.
+
+If this option is set to 0 then pylint will not show errors at bottom.
+
+------------------------------------------------------------------------------
+                                                           *'pymode_lint_signs'*
+Values: 0 or 1.
+Default: 1.
+
+If this option is set to 0 then pylint will not place error signs.
+
+------------------------------------------------------------------------------
+                                                            *'pymode_lint_jump'*
+Values: 0 or 1.
+Default: 0.
+
+If this option is set to 0 then pylint will not jump to the first error.
+
+------------------------------------------------------------------------------
+                                                            *'pymode_lint_hold'*
+Values: 0 or 1.
+Default: 0.
+
+If this option is set to 0 then pylint will switch on the quickfix window when
+it opens. Doesn't work when |'pymode_lint_jump'| enabled.
+
+------------------------------------------------------------------------------
+                                                       *'pymode_lint_minheight'*
+Values: int
+Default: 3.
+
+Set minimal height for the pylint cwindow.
+
+------------------------------------------------------------------------------
+                                               *'pymode_lint_mccabe_complexity'*
+Values: int
+Default: 8.
+
+Set minimal complexity for the mccabe linter.
+
+------------------------------------------------------------------------------
+                                                       *'pymode_lint_maxheight'*
+Values: int
+Default: 6.
+
+Set maximal height for the pylint cwindow.
+
+------------------------------------------------------------------------------
+                                                                 *'pymode_rope'*
+Values: 0 or 1.
+Default: 1.
+
+If this option is set to 0 then the rope script is disabled.
+
+------------------------------------------------------------------------------
+                                                           *'pymode_breakpoint'*
+Values: 0 or 1.
+Default: 1.
+
+If this option is set to 0 then the breakpoint script is disabled.
+
+------------------------------------------------------------------------------
+                                                       *'pymode_breakpoint_key'*
+Default: '<leader>b'.
+
+Key for setting/unsetting breakpoints.
+
+------------------------------------------------------------------------------
+                                                           *'pymode_virtualenv'*
+Values: 0 or 1.
+Default: 1.
+
+If this option is set to 0 then virtualenv support is disabled.
+
+------------------------------------------------------------------------------
+                                                    *'pymode_utils_whitespaces'*
+Values: 0 or 1.
+Default: 1.
+
+Auto-remove unused whitespaces.
+
+------------------------------------------------------------------------------
+                                                               *'pymode_syntax'*
+Values: 0 or 1.
+Default: 1.
+
+If this option is set to 0 then the custom syntax highlighting will
+not be used.
+
+------------------------------------------------------------------------------
+                                                               *'pymode_indent'*
+Values: 0 or 1.
+Default: 1.
+
+If this option is set to 1, pymode will enable python indentation support
+
+------------------------------------------------------------------------------
+                                                              *'pymode_folding'*
+Values: 0 or 1.
+Default: 1.
+
+If this option is set to 1, pymode will enable python-folding.
+
+------------------------------------------------------------------------------
+                                                              *'pymode_options'*
+Values: 0 or 1.
+Default: 1.
+
+If this option is set to 1, pymode will enable the following options for python
+buffers: >
+
+    setlocal complete+=t
+    setlocal formatoptions-=t
+    setlocal number
+    setlocal nowrap
+    setlocal textwidth=80
+<
+------------------------------------------------------------------------------
+                                                               *'pymode_motion'*
+Values: 0 or 1.
+Default: 1.
+
+If this option is set to 1, pymode will enable some python motions.
+Pymode-motion is beta.
+
+================  ============================
+Key               Command
+================  ============================
+[[                Jump to previous class or function (normal, visual, operator modes)
+]]                Jump to next class or function  (normal, visual, operator modes)
+[M                Jump to previous class or method (normal, visual, operator modes)
+]M                Jump to next class or method (normal, visual, operator modes)
+aC                Select a class. Ex: vaC, daC, yaC, caC (normal, operator modes)
+iC                Select inner class. Ex: viC, diC, yiC, ciC (normal, operator modes)
+aM                Select a function or method. Ex: vaM, daM, yaM, caM (normal, operator modes)
+iM                Select inner function or method. Ex: viM, diM, yiM, ciM (normal, operator modes)
+================  ============================
+
+
+==============================================================================
+3. Default Keys ~
+                                                                *PythonModeKeys*
+
+To redefine keys, see: |PythonModeOptions|
+
+================  ============================
+Key               Command
+================  ============================
+K                 Show python docs for current word under cursor
+C-Space           Rope code assist
+<leader>r         Run current buffer
+<leader>b         Set breakpoints
+[[                Jump to previous class or function (normal, visual, operator modes)
+]]                Jump to next class or function  (normal, visual, operator modes)
+[M                Jump to previous class or method (normal, visual, operator modes)
+]M                Jump to next class or method (normal, visual, operator modes)
+aC C              Operation with a class.
+                  Ex: vaC, daC, dC, yaC, yC, caC, cC (normal, operator modes)
+iC                Operation with inner class.
+                  Ex: viC, diC, yiC, ciC (normal, operator modes)
+aM M              Operation with function or method.
+                  Ex: vaM, daM, dM, yaM, yM, caM, cM (normal, operator modes)
+iM                Operation with inner function or method.
+                  Ex: viM, diM, yiM, ciM (normal, operator modes)
+================  ============================
+
+	Note:
+        Also see: |RopeShortcuts|
+
+
+==============================================================================
+4. Commands ~
+                                                            *PythonModeCommands*
+
+*:Pydoc* <args>                                                            *Pydoc*
+    Show python documentation
+
+*:PyLintToggle*                                                     *PyLintToggle*
+    Enable, disable pylint
+
+*:PyLint*                                                                 *PyLint*
+    Check current buffer
+
+*:PyLintAuto*                                                         *PyLintAuto*
+    Automatically fix PEP8 errors in the current buffer
+                                 
+*:Pyrun*                                                                   *Pyrun*
+    Run current buffer
+
+
+==============================================================================
+5. FAQ ~
+                                                                 *PythonModeFAQ*
+
+Python-mode doesn't work
+------------------------
+
+Run ":call pymode#troubleshooting#Test()" and fix the warning or send me the
+output.
+
+
+Rope completion is very slow
+----------------------------
+
+To work, rope_ creates a service directory: `.ropeproject`.  If
+|'pymode_rope_guess_project'| is set on (as it is by default) and
+`.ropeproject` is not found in the current dir, rope will scan for
+`.ropeproject` in every dir in the parent path.  If rope finds `.ropeproject`
+in parent dirs, rope sets project for all child dir and the scan may be slow
+for many dirs and files.
+
+Solutions:
+
+- Disable |'pymode_rope_guess_project'| to make rope always create
+  `.ropeproject` in the current dir.
+- Delete `.ropeproject` from the parent dir to make rope create `.ropeproject`
+  in the current dir.
+- Press `<C-x>po` or `:RopeOpenProject` to force rope to create `.ropeproject`
+  in the current dir.
+
+
+Pylint check is very slow
+-------------------------
+
+In some projects pylint_ may check slowly, because it also scan imported
+modules if possible.  Try using pyflakes: see |'pymode_lint_checker'|.
+
+You may set |exrc| and |secure| in your |vimrc| to auto-set custom settings
+from `.vimrc` from your projects directories.
+>
+    Example: On Flask projects I automatically set 
+             'g:pymode_lint_checker = "pyflakes"'.
+             On Django 'g:pymode_lint_checker = "pylint"'
+<
+
+OSX cannot import urandom
+-------------------------
+
+See: https://groups.google.com/forum/?fromgroups=#!topic/vim_dev/2NXKF6kDONo
+
+The sequence of commands that fixed this:
+>
+    brew unlink python
+    brew unlink macvim
+    brew remove macvim
+    brew install -v --force macvim
+    brew link macvim
+    brew link python
+<
+
+==============================================================================
+6. Credits ~
+                                                             *PythonModeCredits* 
+    Kirill Klenov
+        http://klen.github.com/
+        http://github.com/klen/
+
+    Rope
+        Copyright (C) 2006-2010 Ali Gholami Rudi
+        Copyright (C) 2009-2010 Anton Gritsay
+
+    Pylint
+        Copyright (C) 2003-2011 LOGILAB S.A. (Paris, FRANCE).
+        http://www.logilab.fr/
+
+    Pyflakes:
+        Copyright (c) 2005 Divmod, Inc.
+        http://www.divmod.com/
+
+    PEP8:
+        Copyright (c) 2006 Johann C. Rocholl <johann@rocholl.net>
+        http://github.com/jcrocholl/pep8
+
+    autopep8:
+        Copyright (c) 2012 hhatto <hhatto.jp@gmail.com>
+        https://github.com/hhatto/autopep8
+
+    Python syntax for vim:
+        Copyright (c) 2010 Dmitry Vasiliev
+        http://www.hlabs.spb.ru/vim/python.vim
+
+    PEP8 VIM indentation
+        Copyright (c) 2012 Hynek Schlawack <hs@ox.cx>
+        http://github.com/hynek/vim-python-pep8-indent
+
+
+==============================================================================
+7. License ~
+                                                             *PythonModeLicense*
+
+Python-mode is released under the GNU lesser general public license.
+See: http://www.gnu.org/copyleft/lesser.html
+
+If you like this plugin, you can send me a postcard :) 
+My address is: "Russia, 143401, Krasnogorsk, Shkolnaya 1-19" to "Kirill Klenov".
+Thanks for your support!
+
+
+------------------------------------------------------------------------------
+
+ vim:tw=78:ts=8:ft=help:norl:

doc/ropevim.txt

@@ -0,0 +1,340 @@
+*ropevim.txt*  *Ropevim*  Rope in VIM
+
+==============================================================================
+CONTENTS                                                         *Rope contents*
+
+    1.Refactoring Dialog......................|RopeRefactoringDialog|
+    2.Finding Files...........................|RopeFindingFiles|
+    3.Code Assist.............................|RopeCodeAssist|
+    4.Enabling Autoimport.....................|RopeEnablingAutoimport|
+    5.Filtering Resources.....................|RopeFilteringResources|
+    6.Finding Occurrences.....................|RopeFindOccurrences|
+    7.Dialog Batchset Command.................|RopeDialogBatchsetCommand|
+    8.Variables...............................|RopeVariables|
+    9.Keybindings.............................|RopeKeys|
+
+
+==============================================================================
+1. Refactoring Dialog ~
+                                                         *RopeRefactoringDialog*
+
+Ropevim refactorings use a special kind of dialog.  Depending on the
+refactoring, you'll be asked about the essential information a
+refactoring needs to know (like the new name in rename refactoring).
+
+Next you'll see the base prompt of a refactoring dialog that shows
+something like "Choose what to do".  By entering the name of a
+refactoring option you can set its value.  After setting each option
+you'll be returned back to the base prompt.  Finally, you can ask rope
+to perform, preview or cancel the refactoring.
+
+See |RopeKeys| section and try the refactorings yourself.
+
+
+==============================================================================
+2. Finding Files ~
+                                                              *RopeFindingFiles*
+                                                                 *:RopeFindFile*
+                                                      *:RopeFindFileOtherWindow*
+
+By using |:RopeFindFile| ("<C-x> p f" by default), you can search for
+files in your project.  When you complete the minibuffer you'll see
+all files in the project; files are shown as their reversed paths.
+For instance ``projectroot/docs/todo.txt`` is shown like
+``todo.txt<docs``.  This way you can find files faster in your
+project.  |:RopeFindFileOtherWindow| ("<C-x> p 4 f") opens the
+file in the other window.
+
+
+==============================================================================
+3. Code Assist ~
+                                                              *RopeCodeAssist*
+                                                             *:RopeCodeAssist*
+                                                            *:RopeLuckyAssist*
+                                                    *'pymode_rope_vim_completion'*
+                                                 *'pymode_rope_extended_complete'*
+
+|:RopeCodeAssist| command (<M-/>) will let you select from a list
+of completions.  |:RopeLuckyAssist| command (<M-?>) does not ask
+anything; instead, it inserts the first proposal.
+
+You can tell ropevim to use vim's complete function in insert mode;
+Add: >
+
+  let pymode_rope_vim_completion=1
+<
+to your '~/.vimrc' file.
+
+        Note:
+        That when this variable is set, autoimport completions no longer
+        work since they need to insert an import to the top of the module,
+        too.
+
+By default autocomplete feature will use plain list of proposed completion
+items. You can enable showing extended information about completion
+proposals by setting : >
+
+  let pymode_rope_extended_complete=1
+<
+Completion menu list will show the proposed name itself, one letter which
+shows where this proposal came from (it can be "L" for locals, "G" for
+globals, "B" for builtins, or empty string if such scope definition is not
+applicable), a short object type description (such as "func", "param",
+"meth" and so forth) and a first line of proposed object's docstring (if it
+has one). For function's keyword parameters the last field shows "*" symbol
+if this param is required or "= <default value>" if it is not.
+
+
+==============================================================================
+4. Enabling Autoimport ~
+                                                        *RopeEnablingAutoimport*
+                                                            *:RopevimAutoImport*
+                                                  *:RopeGenerateAutoimportCache*
+
+Rope can propose and automatically import global names in other
+modules.  Rope maintains a cache of global names for each project.  It
+updates the cache only when modules are changed; if you want to cache
+all your modules at once, use |:RopeGenerateAutoimportCache|.  It
+will cache all of the modules inside the project plus those whose
+names are listed in |'pymode_rope_autoimport_modules'| list: >
+
+  " add the name of modules you want to autoimport
+  let g:pymode_rope_autoimport_modules = ["os", "shutil"]
+<
+Now if you are in a buffer that contains: >
+
+  rmtree
+<
+
+and you execute |:RopevimAutoImport| you'll end up with: >
+
+  from shutil import rmtree
+  rmtree
+<
+Also |:RopeCodeAssist| and |:RopeLuckyAssist| propose auto-imported
+names by using "name : module" style.  Selecting them will import
+the module automatically.
+
+
+==============================================================================
+5. Filtering Resources ~
+                                                        *RopeFilteringResources*
+
+Some refactorings, restructuring and find occurrences take an option
+called resources.  This option can be used to limit the resources on
+which a refactoring should be applied.
+
+It uses a simple format: each line starts with either '+' or '-'.
+Each '+' means include the file (or its children if it's a folder)
+that comes after it.  '-' has the same meaning for exclusion.  So
+using: >
+
+  +rope
+  +ropetest
+  -rope/contrib
+<
+means include all python files inside ``rope`` and ``ropetest``
+folders and their subfolder, but those that are in ``rope/contrib``.
+Or: >
+
+  -ropetest
+  -setup.py
+<
+means include all python files inside the project but ``setup.py`` and
+those under ``ropetest`` folder.
+
+
+==============================================================================
+6. Finding Occurrences ~
+                                                           *RopeFindOccurrences*
+
+The find occurrences command ("<C-c> f" by default) can be used to
+find the occurrences of a python name.  If ``unsure`` option is
+``yes``, it will also show unsure occurrences; unsure occurrences are
+indicated with a ``?`` mark in the end. 
+
+        Note:
+        That ropevim uses the quickfix feature of vim for
+        marking occurrence locations.
+
+
+==============================================================================
+7. Dialog Batchset Command ~
+                                                     *RopeDialogBatchsetCommand*
+
+When you use ropevim dialogs there is a command called ``batchset``.
+It can set many options at the same time.  After selecting this
+command from dialog base prompt, you are asked to enter a string.
+
+``batchset`` strings can set the value of configs in two ways.  The
+single line form is like this: >
+
+  name1 value1
+  name2 value2
+<
+
+That is the name of config is followed its value.  For multi-line
+values you can use: >
+
+  name1
+   line1
+   line2
+
+  name2
+   line3
+<
+Each line of the definition should start with a space or a tab. 
+    Note:
+    That blank lines before the name of config definitions are ignored.
+
+``batchset`` command is useful when performing refactorings with long
+configs, like restructurings: >
+
+  pattern ${pycore}.create_module(${project}.root, ${name})
+
+  goal generate.create_module(${project}, ${name})
+
+  imports
+   from rope.contrib import generate
+
+  args
+   pycore: type=rope.base.pycore.PyCore
+   project: type=rope.base.project.Project
+<
+.. ignore the two-space indents
+
+This is a valid ``batchset`` string for restructurings.
+
+Just for the sake of completeness, the reverse of the above
+restructuring can be: >
+
+  pattern ${create_module}(${project}, ${name})
+
+  goal ${project}.pycore.create_module(${project}.root, ${name})
+
+  args
+   create_module: name=rope.contrib.generate.create_module
+   project: type=rope.base.project.Project
+<
+
+==============================================================================
+8. Variables ~
+                                                                 *RopeVariables*
+
+*'pymode_rope_codeassist_maxfixes'*     The maximum number of syntax errors
+                                  to fix for code assists. 
+                                  The default value is `1`.
+
+*'pymode_rope_local_prefix'*            The prefix for ropevim refactorings.
+                                  Defaults to `<C-c> r`.
+
+*'pymode_rope_global_prefix'*           The prefix for ropevim project commands
+                                  Defaults to `<C-x> p`.
+
+*'pymode_rope_enable_shortcuts'*        Shows whether to bind ropevim shortcuts keys.
+                                  Defaults to `1`.
+
+*'pymode_rope_guess_project'*           If non-zero, ropevim tries to guess and
+                                  open the project that contains the file on which
+                                  a ropevim command is performed when no project
+                                  is already open.
+
+*'pymode_rope_enable_autoimport'*       Shows whether to enable autoimport.
+
+*'pymode_rope_autoimport_modules'*      The name of modules whose global names should
+                                  be cached. |:RopeGenerateAutoimportCache| reads
+                                  this list and fills its cache.
+
+*'pymode_rope_autoimport_underlineds'*  If set, autoimport will cache names starting
+                                  with underlines, too.
+
+*'pymode_rope_goto_def_newwin'*         If set, ropevim will open a new buffer
+                                  for "go to definition" result if the definition
+                                  found is located in another file. By default the
+                                  file is open in the same buffer.
+                                  Values: '' -- same buffer, 'new' --
+                                  horizontally split, 'vnew' --
+                                  vertically split
+
+*'pymode_rope_always_show_complete_menu'*   If set, rope autocompletion menu
+always show.
+
+
+==============================================================================
+9. Keybinding ~
+                                                                      *RopeKeys*
+
+Uses almost the same keybinding as ropemacs.
+        Note:
+        That global commands have a `<C-x> p` prefix and local commands
+        have a ``<C-c> r`` prefix.
+        You can change that (see |RopeVariables| section).
+
+
+================  ============================
+Key               Command
+================  ============================
+C-x p o           |:RopeOpenProject|
+C-x p k           |:RopeCloseProject|
+C-x p f           |:RopeFindFile|
+C-x p 4 f         |:RopeFindFileOtherWindow|
+C-x p u           |:RopeUndo|
+C-x p r           |:RopeRedo|
+C-x p c           |:RopeProjectConfig|
+C-x p n [mpfd]    |:RopeCreate|(Module|Package|File|Directory)
+                  |:RopeWriteProject|
+
+C-c r r           |:RopeRename|
+C-c r l           |:RopeExtractVariable|
+C-c r m           |:RopeExtractMethod|
+C-c r i           |:RopeInline|
+C-c r v           |:RopeMove|
+C-c r x           |:RopeRestructure|
+C-c r u           |:RopeUseFunction|
+C-c r f           |:RopeIntroduceFactory|
+C-c r s           |:RopeChangeSignature|
+C-c r 1 r         |:RopeRenameCurrentModule|
+C-c r 1 v         |:RopeMoveCurrentModule|
+C-c r 1 p         |:RopeModuleToPackage|
+
+C-c r o           |:RopeOrganizeImports|
+C-c r n [vfcmp]   |:RopeGenerate|(Variable|Function|Class|Module|Package)
+
+C-c r a /         |:RopeCodeAssist|
+C-c r a g         |:RopeGotoDefinition|
+C-c r a d         |:RopeShowDoc|
+C-c r a f         |:RopeFindOccurrences|
+C-c r a ?         |:RopeLuckyAssist|
+C-c r a j         |:RopeJumpToGlobal|
+C-c r a c         |:RopeShowCalltip|
+                  |:RopeAnalyzeModule|
+
+                  |:RopeAutoImport|
+                  |:RopeGenerateAutoimportCache|
+===============   ============================
+
+
+==============================================================================
+10. Shortcuts ~
+                                                                 *RopeShortcuts*
+
+Some commands are used very frequently; specially the commands in
+code-assist group.  You can define your own shortcuts like this: >
+
+  :map <C-c>g :call RopeGotoDefinition()
+
+<
+
+================  ============================
+Key               Command
+================  ============================
+<C-Space>         |:RopeCodeAssist|
+<C-?>             |:RopeLuckyAssist|
+<C-c> g           |:RopeGotoDefinition|
+<C-c> d           |:RopeShowDoc|
+<C-c> f           |:RopeFindOccurrences|
+================  ============================
+
+------------------------------------------------------------------------------
+
+ vim:tw=78:fo=tcq2:isk=!-~,^*,^\|,^\":ts=8:ft=help:norl:

doc/snipMate.txt

@@ -0,0 +1,322 @@
+*snipMate.txt*  Plugin for using TextMate-style snippets in Vim.
+
+snipMate                                       *snippet* *snippets* *snipMate*
+Last Change: December 27, 2009
+
+|snipMate-description|   Description
+|snipMate-syntax|        Snippet syntax
+|snipMate-usage|         Usage
+|snipMate-settings|      Settings
+|snipMate-features|      Features
+|snipMate-disadvantages| Disadvantages to TextMate
+|snipMate-contact|       Contact
+|snipMate-license|       License
+
+For Vim version 7.0 or later.
+This plugin only works if 'compatible' is not set.
+{Vi does not have any of these features.}
+
+==============================================================================
+DESCRIPTION                                             *snipMate-description*
+
+snipMate.vim implements some of TextMate's snippets features in Vim. A
+snippet is a piece of often-typed text that you can insert into your
+document using a trigger word followed by a <tab>.
+
+For instance, in a C file using the default installation of snipMate.vim, if
+you type "for<tab>" in insert mode, it will expand a typical for loop in C: >
+
+ for (i = 0; i < count; i++) {
+
+ }
+
+
+To go to the next item in the loop, simply <tab> over to it; if there is
+repeated code, such as the "i" variable in this example, you can simply
+start typing once it's highlighted and all the matches specified in the
+snippet will be updated. To go in reverse, use <shift-tab>.
+
+==============================================================================
+SYNTAX                                                        *snippet-syntax*
+
+Snippets can be defined in two ways. They can be in their own file, named
+after their trigger in 'snippets/<filetype>/<trigger>.snippet', or they can be
+defined together in a 'snippets/<filetype>.snippets' file. Note that dotted
+'filetype' syntax is supported -- e.g., you can use >
+
+	:set ft=html.eruby
+
+to activate snippets for both HTML and eRuby for the current file.
+
+The syntax for snippets in *.snippets files is the following: >
+
+ snippet trigger
+ 	expanded text
+	more expanded text
+
+Note that the first hard tab after the snippet trigger is required, and not
+expanded in the actual snippet. The syntax for *.snippet files is the same,
+only without the trigger declaration and starting indentation.
+
+Also note that snippets must be defined using hard tabs. They can be expanded
+to spaces later if desired (see |snipMate-indenting|).
+
+"#" is used as a line-comment character in *.snippets files; however, they can
+only be used outside of a snippet declaration. E.g.: >
+
+ # this is a correct comment
+ snippet trigger
+ 	expanded text
+ snippet another_trigger
+ 	# this isn't a comment!
+	expanded text
+<
+This should hopefully be obvious with the included syntax highlighting.
+
+                                                               *snipMate-${#}*
+Tab stops ~
+
+By default, the cursor is placed at the end of a snippet. To specify where the
+cursor is to be placed next, use "${#}", where the # is the number of the tab
+stop. E.g., to place the cursor first on the id of a <div> tag, and then allow
+the user to press <tab> to go to the middle of it:
+ >
+ snippet div
+ 	<div id="${1}">
+		${2}
+	</div>
+<
+                        *snipMate-placeholders* *snipMate-${#:}* *snipMate-$#*
+Placeholders ~
+
+Placeholder text can be supplied using "${#:text}", where # is the number of
+the tab stop. This text then can be copied throughout the snippet using "$#",
+given # is the same number as used before. So, to make a C for loop: >
+
+ snippet for
+ 	for (${2:i}; $2 < ${1:count}; $1++) {
+		${4}
+	}
+
+This will cause "count" to first be selected and change if the user starts
+typing. When <tab> is pressed, the "i" in ${2}'s position will be selected;
+all $2 variables will default to "i" and automatically be updated if the user
+starts typing.
+NOTE: "$#" syntax is used only for variables, not for tab stops as in TextMate.
+
+Variables within variables are also possible. For instance: >
+
+ snippet opt
+ 	<option value="${1:option}">${2:$1}</option>
+
+Will, as usual, cause "option" to first be selected and update all the $1
+variables if the user starts typing. Since one of these variables is inside of
+${2}, this text will then be used as a placeholder for the next tab stop,
+allowing the user to change it if he wishes.
+
+To copy a value throughout a snippet without supplying default text, simply
+use the "${#:}" construct without the text; e.g.: >
+
+ snippet foo
+ 	${1:}bar$1
+<                                                          *snipMate-commands*
+Interpolated Vim Script ~
+
+Snippets can also contain Vim script commands that are executed (via |eval()|)
+when the snippet is inserted. Commands are given inside backticks (`...`); for
+TextMates's functionality, use the |system()| function. E.g.: >
+
+ snippet date
+ 	`system("date +%Y-%m-%d")`
+
+will insert the current date, assuming you are on a Unix system. Note that you
+can also (and should) use |strftime()| for this example.
+
+Filename([{expr}] [, {defaultText}])             *snipMate-filename* *Filename()*
+
+Since the current filename is used often in snippets, a default function
+has been defined for it in snipMate.vim, appropriately called Filename().
+
+With no arguments, the default filename without an extension is returned;
+the first argument specifies what to place before or after the filename,
+and the second argument supplies the default text to be used if the file
+has not been named. "$1" in the first argument is replaced with the filename;
+if you only want the filename to be returned, the first argument can be left
+blank. Examples: >
+
+ snippet filename
+ 	`Filename()`
+ snippet filename_with_default
+ 	`Filename('', 'name')`
+ snippet filename_foo
+ 	`filename('$1_foo')`
+
+The first example returns the filename if it the file has been named, and an
+empty string if it hasn't. The second returns the filename if it's been named,
+and "name" if it hasn't. The third returns the filename followed by "_foo" if
+it has been named, and an empty string if it hasn't.
+
+                                                                   *multi_snip*
+To specify that a snippet can have multiple matches in a *.snippets file, use
+this syntax: >
+
+ snippet trigger A description of snippet #1
+ 	expand this text
+ snippet trigger A description of snippet #2
+ 	expand THIS text!
+
+In this example, when "trigger<tab>" is typed, a numbered menu containing all
+of the descriptions of the "trigger" will be shown; when the user presses the
+corresponding number, that snippet will then be expanded.
+
+To create a snippet with multiple matches using *.snippet files,
+simply place all the snippets in a subdirectory with the trigger name:
+'snippets/<filetype>/<trigger>/<name>.snippet'.
+
+==============================================================================
+USAGE                                                         *snipMate-usage*
+
+                                                 *'snippets'* *g:snippets_dir*
+Snippets are by default looked for any 'snippets' directory in your
+'runtimepath'. Typically, it is located at '~/.vim/snippets/' on *nix or
+'$HOME\vimfiles\snippets\' on Windows. To change that location or add another
+one, change the g:snippets_dir variable in your |.vimrc| to your preferred
+directory, or use the |ExtractSnips()|function. This will be used by the
+|globpath()| function, and so accepts the same syntax as it (e.g.,
+comma-separated paths).
+
+ExtractSnipsFile({directory}, {filetype})     *ExtractSnipsFile()* *.snippets*
+
+ExtractSnipsFile() extracts the specified *.snippets file for the given
+filetype. A .snippets file contains multiple snippet declarations for the
+filetype. It is further explained above, in |snippet-syntax|.
+
+ExtractSnips({directory}, {filetype})             *ExtractSnips()* *.snippet*
+
+ExtractSnips() extracts *.snippet files from the specified directory and
+defines them as snippets for the given filetype. The directory tree should
+look like this: 'snippets/<filetype>/<trigger>.snippet'. If the snippet has
+multiple matches, it should look like this:
+'snippets/<filetype>/<trigger>/<name>.snippet' (see |multi_snip|).
+
+ResetAllSnippets()                                       *ResetAllSnippets()*
+ResetAllSnippets() removes all snippets from memory. This is useful to put at
+the top of a snippet setup file for if you would like to |:source| it multiple
+times.
+
+ResetSnippets({filetype})                                   *ResetSnippets()*
+ResetSnippets() removes all snippets from memory for the given filetype.
+
+ReloadAllSnippets()                                     *ReloadAllSnippets()*
+ReloadAllSnippets() reloads all snippets for all filetypes. This is useful for
+testing and debugging.
+
+ReloadSnippets({filetype})                                 *ReloadSnippets()*
+ReloadSnippets() reloads all snippets for the given filetype.
+
+                                             *list-snippets* *i_CTRL-R_<Tab>*
+If you would like to see what snippets are available, simply type <c-r><tab>
+in the current buffer to show a list via |popupmenu-completion|.
+
+==============================================================================
+SETTINGS                                  *snipMate-settings* *g:snips_author*
+
+The g:snips_author string (similar to $TM_FULLNAME in TextMate) should be set
+to your name; it can then be used in snippets to automatically add it. E.g.: >
+
+ let g:snips_author = 'Hubert Farnsworth'
+ snippet name
+ 	`g:snips_author`
+<
+                                     *snipMate-expandtab* *snipMate-indenting*
+If you would like your snippets to be expanded using spaces instead of tabs,
+just enable 'expandtab' and set 'softtabstop' to your preferred amount of
+spaces. If 'softtabstop' is not set, 'shiftwidth' is used instead.
+
+                                                              *snipMate-remap*
+snipMate does not come with a setting to customize the trigger key, but you
+can remap it easily in the two lines it's defined in the 'after' directory
+under 'plugin/snipMate.vim'. For instance, to change the trigger key
+to CTRL-J, just change this: >
+
+ ino <tab> <c-r>=TriggerSnippet()<cr>
+ snor <tab> <esc>i<right><c-r>=TriggerSnippet()<cr>
+
+to this: >
+ ino <c-j> <c-r>=TriggerSnippet()<cr>
+ snor <c-j> <esc>i<right><c-r>=TriggerSnippet()<cr>
+
+==============================================================================
+FEATURES                                                   *snipMate-features*
+
+snipMate.vim has the following features among others:
+  - The syntax of snippets is very similar to TextMate's, allowing
+    easy conversion.
+  - The position of the snippet is kept transparently (i.e. it does not use
+    markers/placeholders written to the buffer), which allows you to escape
+    out of an incomplete snippet, something particularly useful in Vim.
+  - Variables in snippets are updated as-you-type.
+  - Snippets can have multiple matches.
+  - Snippets can be out of order. For instance, in a do...while loop, the
+    condition can be added before the code.
+  - [New] File-based snippets are supported.
+  - [New] Triggers after non-word delimiters are expanded, e.g. "foo"
+    in "bar.foo".
+  - [New] <shift-tab> can now be used to jump tab stops in reverse order.
+
+==============================================================================
+DISADVANTAGES                                         *snipMate-disadvantages*
+
+snipMate.vim currently has the following disadvantages to TextMate's snippets:
+    - There is no $0; the order of tab stops must be explicitly stated.
+    - Placeholders within placeholders are not possible. E.g.: >
+
+      '<div${1: id="${2:some_id}}">${3}</div>'
+<
+      In TextMate this would first highlight ' id="some_id"', and if
+      you hit delete it would automatically skip ${2} and go to ${3}
+      on the next <tab>, but if you didn't delete it it would highlight
+      "some_id" first. You cannot do this in snipMate.vim.
+    - Regex cannot be performed on variables, such as "${1/.*/\U&}"
+    - Placeholders cannot span multiple lines.
+    - Activating snippets in different scopes of the same file is
+      not possible.
+
+Perhaps some of these features will be added in a later release.
+
+==============================================================================
+CONTACT                                   *snipMate-contact* *snipMate-author*
+
+To contact the author (Michael Sanders), please email:
+ msanders42+snipmate <at> gmail <dot> com
+
+I greatly appreciate any suggestions or improvements offered for the script.
+
+==============================================================================
+LICENSE                                                     *snipMate-license*
+
+snipMate is released under the MIT license:
+
+Copyright 2009-2010 Michael Sanders. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+The software is provided "as is", without warranty of any kind, express or
+implied, including but not limited to the warranties of merchantability,
+fitness for a particular purpose and noninfringement. In no event shall the
+authors or copyright holders be liable for any claim, damages or other
+liability, whether in an action of contract, tort or otherwise, arising from,
+out of or in connection with the software or the use or other dealings in the
+software.
+
+==============================================================================
+
+vim:tw=78:ts=8:ft=help:norl:

doc/surround.txt

@@ -0,0 +1 @@
+/usr/share/vim-scripts/doc/surround.txt

doc/taglist.txt

@@ -0,0 +1 @@
+/usr/share/vim-scripts/doc/taglist.txt

doc/tags

@@ -0,0 +1,252 @@
+'Tlist_Auto_Highlight_Tag'	taglist.txt	/*'Tlist_Auto_Highlight_Tag'*
+'Tlist_Auto_Open'	taglist.txt	/*'Tlist_Auto_Open'*
+'Tlist_Auto_Update'	taglist.txt	/*'Tlist_Auto_Update'*
+'Tlist_Close_On_Select'	taglist.txt	/*'Tlist_Close_On_Select'*
+'Tlist_Compact_Format'	taglist.txt	/*'Tlist_Compact_Format'*
+'Tlist_Ctags_Cmd'	taglist.txt	/*'Tlist_Ctags_Cmd'*
+'Tlist_Display_Prototype'	taglist.txt	/*'Tlist_Display_Prototype'*
+'Tlist_Display_Tag_Scope'	taglist.txt	/*'Tlist_Display_Tag_Scope'*
+'Tlist_Enable_Fold_Column'	taglist.txt	/*'Tlist_Enable_Fold_Column'*
+'Tlist_Exit_OnlyWindow'	taglist.txt	/*'Tlist_Exit_OnlyWindow'*
+'Tlist_File_Fold_Auto_Close'	taglist.txt	/*'Tlist_File_Fold_Auto_Close'*
+'Tlist_GainFocus_On_ToggleOpen'	taglist.txt	/*'Tlist_GainFocus_On_ToggleOpen'*
+'Tlist_Highlight_Tag_On_BufEnter'	taglist.txt	/*'Tlist_Highlight_Tag_On_BufEnter'*
+'Tlist_Inc_Winwidth'	taglist.txt	/*'Tlist_Inc_Winwidth'*
+'Tlist_Max_Submenu_Items'	taglist.txt	/*'Tlist_Max_Submenu_Items'*
+'Tlist_Max_Tag_Length'	taglist.txt	/*'Tlist_Max_Tag_Length'*
+'Tlist_Process_File_Always'	taglist.txt	/*'Tlist_Process_File_Always'*
+'Tlist_Show_Menu'	taglist.txt	/*'Tlist_Show_Menu'*
+'Tlist_Show_One_File'	taglist.txt	/*'Tlist_Show_One_File'*
+'Tlist_Sort_Type'	taglist.txt	/*'Tlist_Sort_Type'*
+'Tlist_Use_Horiz_Window'	taglist.txt	/*'Tlist_Use_Horiz_Window'*
+'Tlist_Use_Right_Window'	taglist.txt	/*'Tlist_Use_Right_Window'*
+'Tlist_Use_SingleClick'	taglist.txt	/*'Tlist_Use_SingleClick'*
+'Tlist_WinHeight'	taglist.txt	/*'Tlist_WinHeight'*
+'Tlist_WinWidth'	taglist.txt	/*'Tlist_WinWidth'*
+'pymode_breakpoint'	pymode.txt	/*'pymode_breakpoint'*
+'pymode_breakpoint_key'	pymode.txt	/*'pymode_breakpoint_key'*
+'pymode_doc'	pymode.txt	/*'pymode_doc'*
+'pymode_doc_key'	pymode.txt	/*'pymode_doc_key'*
+'pymode_folding'	pymode.txt	/*'pymode_folding'*
+'pymode_indent'	pymode.txt	/*'pymode_indent'*
+'pymode_lint'	pymode.txt	/*'pymode_lint'*
+'pymode_lint_checker'	pymode.txt	/*'pymode_lint_checker'*
+'pymode_lint_config'	pymode.txt	/*'pymode_lint_config'*
+'pymode_lint_cwindow'	pymode.txt	/*'pymode_lint_cwindow'*
+'pymode_lint_hold'	pymode.txt	/*'pymode_lint_hold'*
+'pymode_lint_ignore'	pymode.txt	/*'pymode_lint_ignore'*
+'pymode_lint_jump'	pymode.txt	/*'pymode_lint_jump'*
+'pymode_lint_maxheight'	pymode.txt	/*'pymode_lint_maxheight'*
+'pymode_lint_mccabe_complexity'	pymode.txt	/*'pymode_lint_mccabe_complexity'*
+'pymode_lint_message'	pymode.txt	/*'pymode_lint_message'*
+'pymode_lint_minheight'	pymode.txt	/*'pymode_lint_minheight'*
+'pymode_lint_onfly'	pymode.txt	/*'pymode_lint_onfly'*
+'pymode_lint_select'	pymode.txt	/*'pymode_lint_select'*
+'pymode_lint_signs'	pymode.txt	/*'pymode_lint_signs'*
+'pymode_lint_write'	pymode.txt	/*'pymode_lint_write'*
+'pymode_motion'	pymode.txt	/*'pymode_motion'*
+'pymode_options'	pymode.txt	/*'pymode_options'*
+'pymode_paths'	pymode.txt	/*'pymode_paths'*
+'pymode_rope'	pymode.txt	/*'pymode_rope'*
+'pymode_rope_always_show_complete_menu'	ropevim.txt	/*'pymode_rope_always_show_complete_menu'*
+'pymode_rope_autoimport_modules'	ropevim.txt	/*'pymode_rope_autoimport_modules'*
+'pymode_rope_autoimport_underlineds'	ropevim.txt	/*'pymode_rope_autoimport_underlineds'*
+'pymode_rope_codeassist_maxfixes'	ropevim.txt	/*'pymode_rope_codeassist_maxfixes'*
+'pymode_rope_enable_autoimport'	ropevim.txt	/*'pymode_rope_enable_autoimport'*
+'pymode_rope_enable_shortcuts'	ropevim.txt	/*'pymode_rope_enable_shortcuts'*
+'pymode_rope_extended_complete'	ropevim.txt	/*'pymode_rope_extended_complete'*
+'pymode_rope_global_prefix'	ropevim.txt	/*'pymode_rope_global_prefix'*
+'pymode_rope_goto_def_newwin'	ropevim.txt	/*'pymode_rope_goto_def_newwin'*
+'pymode_rope_guess_project'	ropevim.txt	/*'pymode_rope_guess_project'*
+'pymode_rope_local_prefix'	ropevim.txt	/*'pymode_rope_local_prefix'*
+'pymode_rope_vim_completion'	ropevim.txt	/*'pymode_rope_vim_completion'*
+'pymode_run'	pymode.txt	/*'pymode_run'*
+'pymode_run_key'	pymode.txt	/*'pymode_run_key'*
+'pymode_syntax'	pymode.txt	/*'pymode_syntax'*
+'pymode_utils_whitespaces'	pymode.txt	/*'pymode_utils_whitespaces'*
+'pymode_virtualenv'	pymode.txt	/*'pymode_virtualenv'*
+'snippets'	snipMate.txt	/*'snippets'*
+.snippet	snipMate.txt	/*.snippet*
+.snippets	snipMate.txt	/*.snippets*
+:PyLint	pymode.txt	/*:PyLint*
+:PyLintAuto	pymode.txt	/*:PyLintAuto*
+:PyLintToggle	pymode.txt	/*:PyLintToggle*
+:Pydoc	pymode.txt	/*:Pydoc*
+:Pyrun	pymode.txt	/*:Pyrun*
+:RopeCodeAssist	ropevim.txt	/*:RopeCodeAssist*
+:RopeFindFile	ropevim.txt	/*:RopeFindFile*
+:RopeFindFileOtherWindow	ropevim.txt	/*:RopeFindFileOtherWindow*
+:RopeGenerateAutoimportCache	ropevim.txt	/*:RopeGenerateAutoimportCache*
+:RopeLuckyAssist	ropevim.txt	/*:RopeLuckyAssist*
+:RopevimAutoImport	ropevim.txt	/*:RopevimAutoImport*
+:TlistAddFiles	taglist.txt	/*:TlistAddFiles*
+:TlistAddFilesRecursive	taglist.txt	/*:TlistAddFilesRecursive*
+:TlistClose	taglist.txt	/*:TlistClose*
+:TlistDebug	taglist.txt	/*:TlistDebug*
+:TlistHighlightTag	taglist.txt	/*:TlistHighlightTag*
+:TlistLock	taglist.txt	/*:TlistLock*
+:TlistMessages	taglist.txt	/*:TlistMessages*
+:TlistOpen	taglist.txt	/*:TlistOpen*
+:TlistSessionLoad	taglist.txt	/*:TlistSessionLoad*
+:TlistSessionSave	taglist.txt	/*:TlistSessionSave*
+:TlistShowPrototype	taglist.txt	/*:TlistShowPrototype*
+:TlistShowTag	taglist.txt	/*:TlistShowTag*
+:TlistToggle	taglist.txt	/*:TlistToggle*
+:TlistUndebug	taglist.txt	/*:TlistUndebug*
+:TlistUnlock	taglist.txt	/*:TlistUnlock*
+:TlistUpdate	taglist.txt	/*:TlistUpdate*
+ConqueTerm	conque_term.txt	/*ConqueTerm*
+ConqueTerm_CWInsert	conque_term.txt	/*ConqueTerm_CWInsert*
+ConqueTerm_CloseOnEnd	conque_term.txt	/*ConqueTerm_CloseOnEnd*
+ConqueTerm_CodePage	conque_term.txt	/*ConqueTerm_CodePage*
+ConqueTerm_Color	conque_term.txt	/*ConqueTerm_Color*
+ConqueTerm_ColorMode	conque_term.txt	/*ConqueTerm_ColorMode*
+ConqueTerm_EscKey	conque_term.txt	/*ConqueTerm_EscKey*
+ConqueTerm_ExecFileKey	conque_term.txt	/*ConqueTerm_ExecFileKey*
+ConqueTerm_FastMode	conque_term.txt	/*ConqueTerm_FastMode*
+ConqueTerm_InsertOnEnter	conque_term.txt	/*ConqueTerm_InsertOnEnter*
+ConqueTerm_PromptRegex	conque_term.txt	/*ConqueTerm_PromptRegex*
+ConqueTerm_PyExe	conque_term.txt	/*ConqueTerm_PyExe*
+ConqueTerm_PyVersion	conque_term.txt	/*ConqueTerm_PyVersion*
+ConqueTerm_ReadUnfocused	conque_term.txt	/*ConqueTerm_ReadUnfocused*
+ConqueTerm_SendFileKey	conque_term.txt	/*ConqueTerm_SendFileKey*
+ConqueTerm_SendFunctionKeys	conque_term.txt	/*ConqueTerm_SendFunctionKeys*
+ConqueTerm_SendVisKey	conque_term.txt	/*ConqueTerm_SendVisKey*
+ConqueTerm_SessionSupport	conque_term.txt	/*ConqueTerm_SessionSupport*
+ConqueTerm_StartMessages	conque_term.txt	/*ConqueTerm_StartMessages*
+ConqueTerm_Syntax	conque_term.txt	/*ConqueTerm_Syntax*
+ConqueTerm_TERM	conque_term.txt	/*ConqueTerm_TERM*
+ConqueTerm_ToggleKey	conque_term.txt	/*ConqueTerm_ToggleKey*
+ExtractSnips()	snipMate.txt	/*ExtractSnips()*
+ExtractSnipsFile()	snipMate.txt	/*ExtractSnipsFile()*
+Filename()	snipMate.txt	/*Filename()*
+PyLint	pymode.txt	/*PyLint*
+PyLintAuto	pymode.txt	/*PyLintAuto*
+PyLintToggle	pymode.txt	/*PyLintToggle*
+Pydoc	pymode.txt	/*Pydoc*
+Pyrun	pymode.txt	/*Pyrun*
+Python-mode-contents	pymode.txt	/*Python-mode-contents*
+PythonMode	pymode.txt	/*PythonMode*
+PythonModeCommands	pymode.txt	/*PythonModeCommands*
+PythonModeCredits	pymode.txt	/*PythonModeCredits*
+PythonModeFAQ	pymode.txt	/*PythonModeFAQ*
+PythonModeKeys	pymode.txt	/*PythonModeKeys*
+PythonModeLicense	pymode.txt	/*PythonModeLicense*
+PythonModeModeline	pymode.txt	/*PythonModeModeline*
+PythonModeOptions	pymode.txt	/*PythonModeOptions*
+PythonModeOptionsDetails	pymode.txt	/*PythonModeOptionsDetails*
+ReloadAllSnippets()	snipMate.txt	/*ReloadAllSnippets()*
+ReloadSnippets()	snipMate.txt	/*ReloadSnippets()*
+ResetAllSnippets()	snipMate.txt	/*ResetAllSnippets()*
+ResetSnippets()	snipMate.txt	/*ResetSnippets()*
+RopeCodeAssist	ropevim.txt	/*RopeCodeAssist*
+RopeDialogBatchsetCommand	ropevim.txt	/*RopeDialogBatchsetCommand*
+RopeEnablingAutoimport	ropevim.txt	/*RopeEnablingAutoimport*
+RopeFilteringResources	ropevim.txt	/*RopeFilteringResources*
+RopeFindOccurrences	ropevim.txt	/*RopeFindOccurrences*
+RopeFindingFiles	ropevim.txt	/*RopeFindingFiles*
+RopeKeys	ropevim.txt	/*RopeKeys*
+RopeRefactoringDialog	ropevim.txt	/*RopeRefactoringDialog*
+RopeShortcuts	ropevim.txt	/*RopeShortcuts*
+RopeVariables	ropevim.txt	/*RopeVariables*
+Ropevim	ropevim.txt	/*Ropevim*
+Tlist_Get_Tag_Prototype_By_Line()	taglist.txt	/*Tlist_Get_Tag_Prototype_By_Line()*
+Tlist_Get_Tagname_By_Line()	taglist.txt	/*Tlist_Get_Tagname_By_Line()*
+Tlist_Set_App()	taglist.txt	/*Tlist_Set_App()*
+Tlist_Update_File_Tags()	taglist.txt	/*Tlist_Update_File_Tags()*
+conque-config-general	conque_term.txt	/*conque-config-general*
+conque-config-keyboard	conque_term.txt	/*conque-config-keyboard*
+conque-config-unix	conque_term.txt	/*conque-config-unix*
+conque-config-windows	conque_term.txt	/*conque-config-windows*
+conque-term-api	conque_term.txt	/*conque-term-api*
+conque-term-bugs	conque_term.txt	/*conque-term-bugs*
+conque-term-close	conque_term.txt	/*conque-term-close*
+conque-term-contribute	conque_term.txt	/*conque-term-contribute*
+conque-term-esc	conque_term.txt	/*conque-term-esc*
+conque-term-events	conque_term.txt	/*conque-term-events*
+conque-term-feedback	conque_term.txt	/*conque-term-feedback*
+conque-term-gen-usage	conque_term.txt	/*conque-term-gen-usage*
+conque-term-get-instance	conque_term.txt	/*conque-term-get-instance*
+conque-term-input-mode	conque_term.txt	/*conque-term-input-mode*
+conque-term-installation	conque_term.txt	/*conque-term-installation*
+conque-term-misc	conque_term.txt	/*conque-term-misc*
+conque-term-open	conque_term.txt	/*conque-term-open*
+conque-term-options	conque_term.txt	/*conque-term-options*
+conque-term-read	conque_term.txt	/*conque-term-read*
+conque-term-register	conque_term.txt	/*conque-term-register*
+conque-term-requirements	conque_term.txt	/*conque-term-requirements*
+conque-term-send	conque_term.txt	/*conque-term-send*
+conque-term-set-callback	conque_term.txt	/*conque-term-set-callback*
+conque-term-setup	conque_term.txt	/*conque-term-setup*
+conque-term-special-keys	conque_term.txt	/*conque-term-special-keys*
+conque-term-subprocess	conque_term.txt	/*conque-term-subprocess*
+conque-term-usage	conque_term.txt	/*conque-term-usage*
+conque-term-windows	conque_term.txt	/*conque-term-windows*
+conque-term-write	conque_term.txt	/*conque-term-write*
+conque-term-writeln	conque_term.txt	/*conque-term-writeln*
+cs	surround.txt	/*cs*
+ds	surround.txt	/*ds*
+g:snippets_dir	snipMate.txt	/*g:snippets_dir*
+g:snips_author	snipMate.txt	/*g:snips_author*
+i_CTRL-G_S	surround.txt	/*i_CTRL-G_S*
+i_CTRL-G_s	surround.txt	/*i_CTRL-G_s*
+i_CTRL-R_<Tab>	snipMate.txt	/*i_CTRL-R_<Tab>*
+list-snippets	snipMate.txt	/*list-snippets*
+multi_snip	snipMate.txt	/*multi_snip*
+pymode.txt	pymode.txt	/*pymode.txt*
+python-mode.txt	pymode.txt	/*python-mode.txt*
+ropevim.txt	ropevim.txt	/*ropevim.txt*
+snipMate	snipMate.txt	/*snipMate*
+snipMate-$#	snipMate.txt	/*snipMate-$#*
+snipMate-${#:}	snipMate.txt	/*snipMate-${#:}*
+snipMate-${#}	snipMate.txt	/*snipMate-${#}*
+snipMate-author	snipMate.txt	/*snipMate-author*
+snipMate-commands	snipMate.txt	/*snipMate-commands*
+snipMate-contact	snipMate.txt	/*snipMate-contact*
+snipMate-description	snipMate.txt	/*snipMate-description*
+snipMate-disadvantages	snipMate.txt	/*snipMate-disadvantages*
+snipMate-expandtab	snipMate.txt	/*snipMate-expandtab*
+snipMate-features	snipMate.txt	/*snipMate-features*
+snipMate-filename	snipMate.txt	/*snipMate-filename*
+snipMate-indenting	snipMate.txt	/*snipMate-indenting*
+snipMate-license	snipMate.txt	/*snipMate-license*
+snipMate-placeholders	snipMate.txt	/*snipMate-placeholders*
+snipMate-remap	snipMate.txt	/*snipMate-remap*
+snipMate-settings	snipMate.txt	/*snipMate-settings*
+snipMate-usage	snipMate.txt	/*snipMate-usage*
+snipMate.txt	snipMate.txt	/*snipMate.txt*
+snippet	snipMate.txt	/*snippet*
+snippet-syntax	snipMate.txt	/*snippet-syntax*
+snippets	snipMate.txt	/*snippets*
+surround	surround.txt	/*surround*
+surround-author	surround.txt	/*surround-author*
+surround-customizing	surround.txt	/*surround-customizing*
+surround-issues	surround.txt	/*surround-issues*
+surround-mappings	surround.txt	/*surround-mappings*
+surround-replacements	surround.txt	/*surround-replacements*
+surround-targets	surround.txt	/*surround-targets*
+surround.txt	surround.txt	/*surround.txt*
+taglist-commands	taglist.txt	/*taglist-commands*
+taglist-debug	taglist.txt	/*taglist-debug*
+taglist-extend	taglist.txt	/*taglist-extend*
+taglist-faq	taglist.txt	/*taglist-faq*
+taglist-functions	taglist.txt	/*taglist-functions*
+taglist-install	taglist.txt	/*taglist-install*
+taglist-internet	taglist.txt	/*taglist-internet*
+taglist-intro	taglist.txt	/*taglist-intro*
+taglist-keys	taglist.txt	/*taglist-keys*
+taglist-license	taglist.txt	/*taglist-license*
+taglist-menu	taglist.txt	/*taglist-menu*
+taglist-options	taglist.txt	/*taglist-options*
+taglist-requirements	taglist.txt	/*taglist-requirements*
+taglist-session	taglist.txt	/*taglist-session*
+taglist-todo	taglist.txt	/*taglist-todo*
+taglist-using	taglist.txt	/*taglist-using*
+taglist.txt	taglist.txt	/*taglist.txt*
+vS	surround.txt	/*vS*
+vgS	surround.txt	/*vgS*
+vs	surround.txt	/*vs*
+yS	surround.txt	/*yS*
+ySS	surround.txt	/*ySS*
+ys	surround.txt	/*ys*
+yss	surround.txt	/*yss*

ftplugin/html_snip_helper.vim

@@ -0,0 +1,10 @@
+" Helper function for (x)html snippets
+if exists('s:did_snip_helper') || &cp || !exists('loaded_snips')
+	finish
+endif
+let s:did_snip_helper = 1
+
+" Automatically closes tag if in xhtml
+fun! Close()
+	return stridx(&ft, 'xhtml') == -1 ? '' : ' /'
+endf

ftplugin/python/pymode.vim

@@ -0,0 +1,154 @@
+if pymode#Default('b:pymode', 1)
+    finish
+endif
+
+
+" Parse pymode modeline
+call pymode#Modeline()
+
+
+" Syntax highlight
+if pymode#Option('syntax')
+    let python_highlight_all=1
+endif
+
+
+" Options {{{
+
+" Python other options
+if pymode#Option('options')
+    setlocal complete+=t
+    setlocal formatoptions-=t
+    if v:version > 702 && !&relativenumber
+        setlocal number
+    endif
+    setlocal nowrap
+    setlocal textwidth=79
+endif
+
+" }}}
+
+
+" Documentation {{{
+
+if pymode#Option('doc')
+
+    " DESC: Set commands
+    command! -buffer -nargs=1 Pydoc call pymode#doc#Show("<args>")
+
+    " DESC: Set keys
+    exe "nnoremap <silent> <buffer> " g:pymode_doc_key ":call pymode#doc#Show(expand('<cword>'))<CR>"
+    exe "vnoremap <silent> <buffer> " g:pymode_doc_key ":<C-U>call pymode#doc#Show(@*)<CR>"
+
+endif
+
+" }}}
+
+
+" Lint {{{
+
+if pymode#Option('lint')
+
+    " DESC: Set commands
+    command! -buffer -nargs=0 PyLintToggle :call pymode#lint#Toggle()
+    command! -buffer -nargs=0 PyLintWindowToggle :call pymode#lint#ToggleWindow()
+    command! -buffer -nargs=0 PyLintCheckerToggle :call pymode#lint#ToggleChecker()
+    command! -buffer -nargs=0 PyLint :call pymode#lint#Check()
+    command! -buffer -nargs=0 PyLintAuto :call pymode#lint#Auto()
+
+    " DESC: Set autocommands
+    if pymode#Option('lint_write')
+        au BufWritePost <buffer> PyLint
+    endif
+
+    if pymode#Option('lint_onfly')
+        au InsertLeave <buffer> PyLint
+    endif
+
+    if pymode#Option('lint_message')
+        au CursorHold <buffer> call pymode#lint#show_errormessage()
+        au CursorMoved <buffer> call pymode#lint#show_errormessage()
+    endif
+
+    " DESC: Run queue
+    let &l:updatetime = g:pymode_updatetime
+    au CursorHold <buffer> call pymode#queue#Poll()
+    au BufLeave <buffer> py queue.stop_queue()
+
+endif
+
+" }}}
+
+
+" Rope {{{
+
+if pymode#Option('rope')
+
+    " DESC: Set keys
+    exe "noremap <silent> <buffer> " . g:pymode_rope_short_prefix . "g :RopeGotoDefinition<CR>"
+    exe "noremap <silent> <buffer> " . g:pymode_rope_short_prefix . "d :RopeShowDoc<CR>"
+    exe "noremap <silent> <buffer> " . g:pymode_rope_short_prefix . "f :RopeFindOccurrences<CR>"
+    exe "noremap <silent> <buffer> " . g:pymode_rope_short_prefix . "m :emenu Rope . <TAB>"
+    inoremap <silent> <buffer> <S-TAB> <C-R>=RopeLuckyAssistInsertMode()<CR>
+
+    if g:pymode_rope_map_space
+        let s:prascm = g:pymode_rope_always_show_complete_menu ? "<C-P>" : ""
+        exe "inoremap <silent> <buffer> <Nul> <C-R>=RopeCodeAssistInsertMode()<CR>" . s:prascm
+        exe "inoremap <silent> <buffer> <c-space> <C-R>=RopeCodeAssistInsertMode()<CR>" . s:prascm
+    endif
+
+endif
+
+" }}}
+
+
+" Execution {{{
+
+if pymode#Option('run')
+
+    " DESC: Set commands
+    command! -buffer -nargs=0 -range=% Pyrun call pymode#run#Run(<f-line1>, <f-line2>)
+
+    " DESC: Set keys
+    exe "nnoremap <silent> <buffer> " g:pymode_run_key ":Pyrun<CR>"
+    exe "vnoremap <silent> <buffer> " g:pymode_run_key ":Pyrun<CR>"
+
+endif
+
+" }}}
+
+
+" Breakpoints {{{
+
+if pymode#Option('breakpoint')
+
+    " DESC: Set keys
+    exe "nnoremap <silent> <buffer> " g:pymode_breakpoint_key ":call pymode#breakpoint#Set(line('.'))<CR>"
+
+endif
+
+" }}}
+
+
+" Utils {{{
+
+if pymode#Option('utils_whitespaces')
+    au BufWritePre <buffer> call pymode#TrimWhiteSpace()
+endif
+
+" }}}
+
+
+" Folding {{{
+
+if pymode#Option('folding')
+
+    setlocal foldmethod=expr
+    setlocal foldexpr=pymode#folding#expr(v:lnum)
+    setlocal foldtext=pymode#folding#text()
+
+endif
+
+" }}}
+
+" vim: fdm=marker:fdl=0

plugin-info.txt

@@ -0,0 +1,8 @@
+{
+  "name" : "snipmate",
+  "version" : "dev",
+  "author" : "Michael Sanders <msanders42@gmail.com>",
+  "repository" : {"type": "git", "url": "git://github.com/msanders/snipmate.vim.git"},
+  "dependencies" : {},
+  "description" : "snipMate.vim aims to be a concise vim script that implements some of TextMate's snippets features in Vim."
+}

plugin/conque_term.vim

@@ -0,0 +1 @@
+/usr/share/vim-conque/plugin/conque_term.vim

plugin/pymode.vim

@@ -0,0 +1,339 @@
+let g:pymode_version = "0.6.9"
+
+com! PymodeVersion echomsg "Current python-mode version: " . g:pymode_version
+
+" OPTION: g:pymode -- bool. Run pymode.
+if pymode#Default('g:pymode', 1) || !g:pymode
+    " DESC: Disable script loading
+    finish
+endif
+
+" DESC: Check python support
+if !has('python')
+    echoerr expand("<sfile>:t") . " required vim compiled with +python."
+    let g:pymode_lint       = 0
+    let g:pymode_rope       = 0
+    let g:pymode_path       = 0
+    let g:pymode_doc        = 0
+    let g:pymode_run        = 0
+    let g:pymode_virtualenv = 0
+endif
+
+
+" Virtualenv {{{
+
+if !pymode#Default("g:pymode_virtualenv", 1) || g:pymode_virtualenv
+
+    call pymode#Default("g:pymode_virtualenv_enabled", [])
+
+    " Add virtualenv paths
+    call pymode#virtualenv#Activate()
+
+endif
+
+" }}}
+
+
+" DESC: Fix python path
+if !pymode#Default('g:pymode_path', 1) || g:pymode_path
+
+    call pymode#Default('g:pymode_paths', [])
+
+python << EOF
+import sys, vim, os
+
+curpath = vim.eval("getcwd()")
+libpath = os.path.join(os.path.dirname(os.path.dirname(
+    vim.eval("expand('<sfile>:p')"))), 'pylibs')
+
+sys.path = [os.path.dirname(libpath), libpath, curpath] + vim.eval("g:pymode_paths") + sys.path
+EOF
+
+endif
+
+
+" Lint {{{
+
+if !pymode#Default("g:pymode_lint", 1) || g:pymode_lint
+
+    let g:qf_list = []
+    let g:pymode_lint_buffer = 0
+
+    " OPTION: g:pymode_lint_write -- bool. Check code every save.
+    call pymode#Default("g:pymode_lint_write", 1)
+
+    " OPTION: g:pymode_lint_onfly -- bool. Check code every save.
+    call pymode#Default("g:pymode_lint_onfly", 0)
+
+    " OPTION: g:pymode_lint_message -- bool. Show current line error message
+    call pymode#Default("g:pymode_lint_message", 1)
+
+    " OPTION: g:pymode_lint_checker -- str. Choices are: pylint, pyflakes, pep8, mccabe
+    call pymode#Default("g:pymode_lint_checker", "pyflakes,pep8,mccabe")
+
+    " OPTION: g:pymode_lint_config -- str. Path to pylint config file
+    call pymode#Default("g:pymode_lint_config", $HOME . "/.pylintrc")
+
+    " OPTION: g:pymode_lint_cwindow -- bool. Auto open cwindow if errors find
+    call pymode#Default("g:pymode_lint_cwindow", 1)
+
+    " OPTION: g:pymode_lint_jump -- int. Jump on first error.
+    call pymode#Default("g:pymode_lint_jump", 0)
+
+    " OPTION: g:pymode_lint_hold -- int. Hold cursor on current window when
+    " quickfix open
+    call pymode#Default("g:pymode_lint_hold", 0)
+
+    " OPTION: g:pymode_lint_minheight -- int. Minimal height of pymode lint window
+    call pymode#Default("g:pymode_lint_minheight", 3)
+
+    " OPTION: g:pymode_lint_maxheight -- int. Maximal height of pymode lint window
+    call pymode#Default("g:pymode_lint_maxheight", 6)
+
+    " OPTION: g:pymode_lint_ignore -- string. Skip errors and warnings (e.g. E4,W)
+    call pymode#Default("g:pymode_lint_ignore", "E501")
+
+    " OPTION: g:pymode_lint_select -- string. Select errors and warnings (e.g. E4,W)
+    call pymode#Default("g:pymode_lint_select", "")
+
+    " OPTION: g:pymode_lint_mccabe_complexity -- int. Maximum allowed complexity
+    call pymode#Default("g:pymode_lint_mccabe_complexity", 8)
+
+    " OPTION: g:pymode_lint_signs_always_visible -- bool. Always show the
+    " errors ruller, even if there's no errors.
+    call pymode#Default("g:pymode_lint_signs_always_visible", 0)
+
+    " OPTION: g:pymode_lint_signs -- bool. Place error signs
+    if (!pymode#Default("g:pymode_lint_signs", 1) || g:pymode_lint_signs) && has('signs')
+
+        " DESC: Signs definition
+        sign define W text=WW texthl=Todo
+        sign define C text=CC texthl=Comment
+        sign define R text=RR texthl=Visual
+        sign define E text=EE texthl=Error
+        sign define I text=II texthl=Info
+
+        if !pymode#Default("g:pymode_lint_signs_always_visible", 0) || g:pymode_lint_signs_always_visible
+            " Show the sign's ruller if asked for, even it there's no error to show
+            sign define __dummy__
+            autocmd BufRead,BufNew * call RopeShowSignsRulerIfNeeded()
+        endif
+
+    endif
+
+    " DESC: Set default pylint configuration
+    if !filereadable(g:pymode_lint_config)
+        let g:pymode_lint_config = expand("<sfile>:p:h:h") . "/pylint.ini"
+    endif
+
+    py from pymode import lint, queue, auto
+
+    au VimLeavePre * py queue.stop_queue()
+
+endif
+
+" }}}
+
+
+" Documentation {{{
+
+if !pymode#Default("g:pymode_doc", 1) || g:pymode_doc
+
+    " OPTION: g:pymode_doc_key -- string. Key for show python documantation.
+    call pymode#Default("g:pymode_doc_key", "K")
+
+endif
+
+" }}}
+
+
+" Breakpoints {{{
+
+if !pymode#Default("g:pymode_breakpoint", 1) || g:pymode_breakpoint
+
+    if !pymode#Default("g:pymode_breakpoint_cmd", "import ipdb; ipdb.set_trace() ### XXX BREAKPOINT")  && has("python")
+python << EOF
+from imp import find_module
+try:
+    find_module('ipdb')
+except ImportError:
+    vim.command('let g:pymode_breakpoint_cmd = "import pdb; pdb.set_trace() ### XXX BREAKPOINT"')
+EOF
+    endif
+
+    " OPTION: g:pymode_breakpoint_key -- string. Key for set/unset breakpoint.
+    call pymode#Default("g:pymode_breakpoint_key", "<leader>b")
+
+endif
+
+" }}}
+
+
+" Execution {{{
+
+if !pymode#Default("g:pymode_run", 1) || g:pymode_run
+
+    " OPTION: g:pymode_doc_key -- string. Key for show python documentation.
+    call pymode#Default("g:pymode_run_key", "<leader>r")
+
+endif
+
+" }}}
+
+
+" Rope {{{
+
+if !pymode#Default("g:pymode_rope", 1) || g:pymode_rope
+
+    " OPTION: g:pymode_rope_auto_project -- bool. Auto create ropeproject
+    call pymode#Default("g:pymode_rope_auto_project", 1)
+
+    " OPTION: g:pymode_rope_auto_project_open -- bool.
+    " Auto open existing projects, ie, if the current directory has a
+    " `.ropeproject` subdirectory.
+    call pymode#Default("g:pymode_rope_auto_project_open", 1)
+
+    " OPTION: g:pymode_rope_auto_session_manage -- bool
+    call pymode#Default("g:pymode_rope_auto_session_manage", 0)
+
+    " OPTION: g:pymode_rope_enable_autoimport -- bool. Enable autoimport
+    call pymode#Default("g:pymode_rope_enable_autoimport", 1)
+
+    " OPTION: g:pymode_rope_autoimport_generate -- bool.
+    call pymode#Default("g:pymode_rope_autoimport_generate", 1)
+
+    " OPTION: g:pymode_rope_autoimport_underlines -- bool.
+    call pymode#Default("g:pymode_rope_autoimport_underlineds", 0)
+
+    " OPTION: g:pymode_rope_codeassist_maxfiles -- bool.
+    call pymode#Default("g:pymode_rope_codeassist_maxfixes", 10)
+
+    " OPTION: g:pymode_rope_sorted_completions -- bool.
+    call pymode#Default("g:pymode_rope_sorted_completions", 1)
+
+    " OPTION: g:pymode_rope_extended_complete -- bool.
+    call pymode#Default("g:pymode_rope_extended_complete", 1)
+
+    " OPTION: g:pymode_rope_autoimport_modules -- array.
+    call pymode#Default("g:pymode_rope_autoimport_modules", ["os","shutil","datetime"])
+
+    " OPTION: g:pymode_rope_confirm_saving -- bool.
+    call pymode#Default("g:pymode_rope_confirm_saving", 1)
+
+    " OPTION: g:pymode_rope_global_prefix -- string.
+    call pymode#Default("g:pymode_rope_global_prefix", "<C-x>p")
+
+    " OPTION: g:pymode_rope_local_prefix -- string.
+    call pymode#Default("g:pymode_rope_local_prefix", "<C-c>r")
+
+    " OPTION: g:pymode_rope_short_prefix -- string.
+    call pymode#Default("g:pymode_rope_short_prefix", "<C-c>")
+
+    " OPTION: g:pymode_rope_map_space -- string.
+    call pymode#Default("g:pymode_rope_map_space", 1)
+
+    " OPTION: g:pymode_rope_vim_completion -- bool.
+    call pymode#Default("g:pymode_rope_vim_completion", 1)
+
+    " OPTION: g:pymode_rope_guess_project -- bool.
+    call pymode#Default("g:pymode_rope_guess_project", 1)
+
+    " OPTION: g:pymode_rope_goto_def_newwin -- str ('new', 'vnew', '').
+    call pymode#Default("g:pymode_rope_goto_def_newwin", "")
+
+    " OPTION: g:pymode_rope_always_show_complete_menu -- bool.
+    call pymode#Default("g:pymode_rope_always_show_complete_menu", 0)
+
+    " DESC: Init Rope
+    py import ropevim
+
+    fun! RopeCodeAssistInsertMode() "{{{
+        call RopeCodeAssist()
+        return ""
+    endfunction "}}}
+
+    fun! RopeOpenExistingProject() "{{{
+        if isdirectory(getcwd() . '/.ropeproject')
+            " In order to pass it the quiet kwarg I need to open the project
+            " using python and not vim, which should be no major issue
+            py ropevim._interface.open_project(quiet=True)
+            return ""
+        endif
+    endfunction "}}}
+
+    fun! RopeLuckyAssistInsertMode() "{{{
+        call RopeLuckyAssist()
+        return ""
+    endfunction "}}}
+
+    fun! RopeOmni(findstart, base) "{{{
+        if a:findstart
+            py ropevim._interface._find_start()
+            return g:pymode_offset
+        else
+            call RopeOmniComplete()
+            return g:pythoncomplete_completions
+        endif
+    endfunction "}}}
+
+    fun! RopeShowSignsRulerIfNeeded() "{{{
+        if &ft == 'python'
+            execute printf('silent! sign place 1 line=1 name=__dummy__ file=%s', expand("%:p"))
+        endif
+     endfunction "}}}
+
+
+    " Rope menu
+    menu <silent> Rope.Autoimport :RopeAutoImport<CR>
+    menu <silent> Rope.ChangeSignature :RopeChangeSignature<CR>
+    menu <silent> Rope.CloseProject :RopeCloseProject<CR>
+    menu <silent> Rope.GenerateAutoImportCache :RopeGenerateAutoimportCache<CR>
+    menu <silent> Rope.ExtractVariable :RopeExtractVariable<CR>
+    menu <silent> Rope.ExtractMethod :RopeExtractMethod<CR>
+    menu <silent> Rope.Inline :RopeInline<CR>
+    menu <silent> Rope.IntroduceFactory :RopeIntroduceFactory<CR>
+    menu <silent> Rope.FindFile :RopeFindFile<CR>
+    menu <silent> Rope.OpenProject :RopeOpenProject<CR>
+    menu <silent> Rope.Move :RopeMove<CR>
+    menu <silent> Rope.MoveCurrentModule :RopeMoveCurrentModule<CR>
+    menu <silent> Rope.ModuleToPackage :RopeModuleToPackage<CR>
+    menu <silent> Rope.Redo :RopeRedo<CR>
+    menu <silent> Rope.Rename :RopeRename<CR>
+    menu <silent> Rope.RenameCurrentModule :RopeRenameCurrentModule<CR>
+    menu <silent> Rope.Restructure :RopeRestructure<CR>
+    menu <silent> Rope.Undo :RopeUndo<CR>
+    menu <silent> Rope.UseFunction :RopeUseFunction<CR>
+
+    if !pymode#Default("g:pymode_rope_auto_project_open", 1) || g:pymode_rope_auto_project_open
+        call RopeOpenExistingProject()
+    endif
+
+     if !pymode#Default("g:pymode_rope_auto_session_manage", 0) || g:pymode_rope_auto_session_manage
+        autocmd VimLeave * call RopeSaveSession()
+        autocmd VimEnter * call RopeRestoreSession()
+     endif
+
+endif
+
+" }}}
+
+
+" OPTION: g:pymode_folding -- bool. Enable python-mode folding for pyfiles.
+call pymode#Default("g:pymode_folding", 1)
+
+" OPTION: g:pymode_syntax -- bool. Enable python-mode syntax for pyfiles.
+call pymode#Default("g:pymode_syntax", 1)
+
+" OPTION: g:pymode_indent -- bool. Enable/Disable pymode PEP8 indentation
+call pymode#Default("g:pymode_indent", 1)
+
+" OPTION: g:pymode_utils_whitespaces -- bool. Remove unused whitespaces on save
+call pymode#Default("g:pymode_utils_whitespaces", 1)
+
+" OPTION: g:pymode_options -- bool. To set some python options.
+call pymode#Default("g:pymode_options", 1)
+
+" OPTION: g:pymode_updatetime -- int. Set updatetime for async pymode's operation
+call pymode#Default("g:pymode_updatetime", 1000)
+
+" vim: fdm=marker:fdl=0

plugin/snipMate.vim

@@ -0,0 +1,271 @@
+" File:          snipMate.vim
+" Author:        Michael Sanders
+" Version:       0.84
+" Description:   snipMate.vim implements some of TextMate's snippets features in
+"                Vim. A snippet is a piece of often-typed text that you can
+"                insert into your document using a trigger word followed by a "<tab>".
+"
+"                For more help see snipMate.txt; you can do this by using:
+"                :helptags ~/.vim/doc
+"                :h snipMate.txt
+
+if exists('loaded_snips') || &cp || version < 700
+	finish
+endif
+let loaded_snips = 1
+if !exists('snips_author') | let snips_author = 'Me' | endif
+
+au BufRead,BufNewFile *.snippets\= set ft=snippet
+au FileType snippet setl noet fdm=indent
+
+let s:snippets = {} | let s:multi_snips = {}
+
+if !exists('snippets_dir')
+	let snippets_dir = substitute(globpath(&rtp, 'snippets/'), "\n", ',', 'g')
+endif
+
+fun! MakeSnip(scope, trigger, content, ...)
+	let multisnip = a:0 && a:1 != ''
+	let var = multisnip ? 's:multi_snips' : 's:snippets'
+	if !has_key({var}, a:scope) | let {var}[a:scope] = {} | endif
+	if !has_key({var}[a:scope], a:trigger)
+		let {var}[a:scope][a:trigger] = multisnip ? [[a:1, a:content]] : a:content
+	elseif multisnip | let {var}[a:scope][a:trigger] += [[a:1, a:content]]
+	else
+		echom 'Warning in snipMate.vim: Snippet '.a:trigger.' is already defined.'
+				\ .' See :h multi_snip for help on snippets with multiple matches.'
+	endif
+endf
+
+fun! ExtractSnips(dir, ft)
+	for path in split(globpath(a:dir, '*'), "\n")
+		if isdirectory(path)
+			let pathname = fnamemodify(path, ':t')
+			for snipFile in split(globpath(path, '*.snippet'), "\n")
+				call s:ProcessFile(snipFile, a:ft, pathname)
+			endfor
+		elseif fnamemodify(path, ':e') == 'snippet'
+			call s:ProcessFile(path, a:ft)
+		endif
+	endfor
+endf
+
+" Processes a single-snippet file; optionally add the name of the parent
+" directory for a snippet with multiple matches.
+fun s:ProcessFile(file, ft, ...)
+	let keyword = fnamemodify(a:file, ':t:r')
+	if keyword  == '' | return | endif
+	try
+		let text = join(readfile(a:file), "\n")
+	catch /E484/
+		echom "Error in snipMate.vim: couldn't read file: ".a:file
+	endtry
+	return a:0 ? MakeSnip(a:ft, a:1, text, keyword)
+			\  : MakeSnip(a:ft, keyword, text)
+endf
+
+fun! ExtractSnipsFile(file, ft)
+	if !filereadable(a:file) | return | endif
+	let text = readfile(a:file)
+	let inSnip = 0
+	for line in text + ["\n"]
+		if inSnip && (line[0] == "\t" || line == '')
+			let content .= strpart(line, 1)."\n"
+			continue
+		elseif inSnip
+			call MakeSnip(a:ft, trigger, content[:-2], name)
+			let inSnip = 0
+		endif
+
+		if line[:6] == 'snippet'
+			let inSnip = 1
+			let trigger = strpart(line, 8)
+			let name = ''
+			let space = stridx(trigger, ' ') + 1
+			if space " Process multi snip
+				let name = strpart(trigger, space)
+				let trigger = strpart(trigger, 0, space - 1)
+			endif
+			let content = ''
+		endif
+	endfor
+endf
+
+" Reset snippets for filetype.
+fun! ResetSnippets(ft)
+	let ft = a:ft == '' ? '_' : a:ft
+	for dict in [s:snippets, s:multi_snips, g:did_ft]
+		if has_key(dict, ft)
+			unlet dict[ft]
+		endif
+	endfor
+endf
+
+" Reset snippets for all filetypes.
+fun! ResetAllSnippets()
+	let s:snippets = {} | let s:multi_snips = {} | let g:did_ft = {}
+endf
+
+" Reload snippets for filetype.
+fun! ReloadSnippets(ft)
+	let ft = a:ft == '' ? '_' : a:ft
+	call ResetSnippets(ft)
+	call GetSnippets(g:snippets_dir, ft)
+endf
+
+" Reload snippets for all filetypes.
+fun! ReloadAllSnippets()
+	for ft in keys(g:did_ft)
+		call ReloadSnippets(ft)
+	endfor
+endf
+
+let g:did_ft = {}
+fun! GetSnippets(dir, filetypes)
+	for ft in split(a:filetypes, '\.')
+		if has_key(g:did_ft, ft) | continue | endif
+		call s:DefineSnips(a:dir, ft, ft)
+		if ft == 'objc' || ft == 'cpp' || ft == 'cs'
+			call s:DefineSnips(a:dir, 'c', ft)
+		elseif ft == 'xhtml'
+			call s:DefineSnips(a:dir, 'html', 'xhtml')
+		endif
+		let g:did_ft[ft] = 1
+	endfor
+endf
+
+" Define "aliasft" snippets for the filetype "realft".
+fun s:DefineSnips(dir, aliasft, realft)
+	for path in split(globpath(a:dir, a:aliasft.'/')."\n".
+					\ globpath(a:dir, a:aliasft.'-*/'), "\n")
+		call ExtractSnips(path, a:realft)
+	endfor
+	for path in split(globpath(a:dir, a:aliasft.'.snippets')."\n".
+					\ globpath(a:dir, a:aliasft.'-*.snippets'), "\n")
+		call ExtractSnipsFile(path, a:realft)
+	endfor
+endf
+
+fun! TriggerSnippet()
+	if exists('g:SuperTabMappingForward')
+		if g:SuperTabMappingForward == "<tab>"
+			let SuperTabKey = "\<c-n>"
+		elseif g:SuperTabMappingBackward == "<tab>"
+			let SuperTabKey = "\<c-p>"
+		endif
+	endif
+
+	if pumvisible() " Update snippet if completion is used, or deal with supertab
+		if exists('SuperTabKey')
+			call feedkeys(SuperTabKey) | return ''
+		endif
+		call feedkeys("\<esc>a", 'n') " Close completion menu
+		call feedkeys("\<tab>") | return ''
+	endif
+
+	if exists('g:snipPos') | return snipMate#jumpTabStop(0) | endif
+
+	let word = matchstr(getline('.'), '\S\+\%'.col('.').'c')
+	for scope in [bufnr('%')] + split(&ft, '\.') + ['_']
+		let [trigger, snippet] = s:GetSnippet(word, scope)
+		" If word is a trigger for a snippet, delete the trigger & expand
+		" the snippet.
+		if snippet != ''
+			let col = col('.') - len(trigger)
+			sil exe 's/\V'.escape(trigger, '/\.').'\%#//'
+			return snipMate#expandSnip(snippet, col)
+		endif
+	endfor
+
+	if exists('SuperTabKey')
+		call feedkeys(SuperTabKey)
+		return ''
+	endif
+	return "\<tab>"
+endf
+
+fun! BackwardsSnippet()
+	if exists('g:snipPos') | return snipMate#jumpTabStop(1) | endif
+
+	if exists('g:SuperTabMappingForward')
+		if g:SuperTabMappingBackward == "<s-tab>"
+			let SuperTabKey = "\<c-p>"
+		elseif g:SuperTabMappingForward == "<s-tab>"
+			let SuperTabKey = "\<c-n>"
+		endif
+	endif
+	if exists('SuperTabKey')
+		call feedkeys(SuperTabKey)
+		return ''
+	endif
+	return "\<s-tab>"
+endf
+
+" Check if word under cursor is snippet trigger; if it isn't, try checking if
+" the text after non-word characters is (e.g. check for "foo" in "bar.foo")
+fun s:GetSnippet(word, scope)
+	let word = a:word | let snippet = ''
+	while snippet == ''
+		if exists('s:snippets["'.a:scope.'"]["'.escape(word, '\"').'"]')
+			let snippet = s:snippets[a:scope][word]
+		elseif exists('s:multi_snips["'.a:scope.'"]["'.escape(word, '\"').'"]')
+			let snippet = s:ChooseSnippet(a:scope, word)
+			if snippet == '' | break | endif
+		else
+			if match(word, '\W') == -1 | break | endif
+			let word = substitute(word, '.\{-}\W', '', '')
+		endif
+	endw
+	if word == '' && a:word != '.' && stridx(a:word, '.') != -1
+		let [word, snippet] = s:GetSnippet('.', a:scope)
+	endif
+	return [word, snippet]
+endf
+
+fun s:ChooseSnippet(scope, trigger)
+	let snippet = []
+	let i = 1
+	for snip in s:multi_snips[a:scope][a:trigger]
+		let snippet += [i.'. '.snip[0]]
+		let i += 1
+	endfor
+	if i == 2 | return s:multi_snips[a:scope][a:trigger][0][1] | endif
+	let num = inputlist(snippet) - 1
+	return num == -1 ? '' : s:multi_snips[a:scope][a:trigger][num][1]
+endf
+
+fun! ShowAvailableSnips()
+	let line  = getline('.')
+	let col   = col('.')
+	let word  = matchstr(getline('.'), '\S\+\%'.col.'c')
+	let words = [word]
+	if stridx(word, '.')
+		let words += split(word, '\.', 1)
+	endif
+	let matchlen = 0
+	let matches = []
+	for scope in [bufnr('%')] + split(&ft, '\.') + ['_']
+		let triggers = has_key(s:snippets, scope) ? keys(s:snippets[scope]) : []
+		if has_key(s:multi_snips, scope)
+			let triggers += keys(s:multi_snips[scope])
+		endif
+		for trigger in triggers
+			for word in words
+				if word == ''
+					let matches += [trigger] " Show all matches if word is empty
+				elseif trigger =~ '^'.word
+					let matches += [trigger]
+					let len = len(word)
+					if len > matchlen | let matchlen = len | endif
+				endif
+			endfor
+		endfor
+	endfor
+
+	" This is to avoid a bug with Vim when using complete(col - matchlen, matches)
+	" (Issue#46 on the Google Code snipMate issue tracker).
+	call setline(line('.'), substitute(line, repeat('.', matchlen).'\%'.col.'c', '', ''))
+	call complete(col, matches)
+	return ''
+endf
+" vim:noet:sw=4:ts=4:ft=vim

plugin/surround.vim

@@ -0,0 +1 @@
+/usr/share/vim-scripts/plugin/surround.vim

plugin/taglist.vim

@@ -0,0 +1 @@
+/usr/share/vim-scripts/plugin/taglist.vim

pylibs/autopep8.py

@@ -0,0 +1,1726 @@
+#!/usr/bin/env python
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+"""
+A tool that automatically formats Python code to conform to the PEP 8 style
+guide.
+"""
+from __future__ import print_function
+
+import copy
+import os
+import re
+import sys
+import inspect
+import codecs
+import locale
+try:
+    from StringIO import StringIO
+except ImportError:
+    from io import StringIO
+import token
+import tokenize
+from optparse import OptionParser
+from subprocess import Popen, PIPE
+from difflib import unified_diff
+import tempfile
+
+from distutils.version import StrictVersion
+try:
+    import pep8
+    if StrictVersion(pep8.__version__) < StrictVersion('1.3a2'):
+        pep8 = None
+except ImportError:
+    pep8 = None
+
+
+__version__ = '0.8.1'
+
+
+PEP8_BIN = 'pep8'
+CR = '\r'
+LF = '\n'
+CRLF = '\r\n'
+MAX_LINE_WIDTH = 79
+
+
+def open_with_encoding(filename, encoding, mode='r'):
+    """Return opened file with a specific encoding."""
+    try:
+        # Python 3
+        return open(filename, mode=mode, encoding=encoding)
+    except TypeError:
+        # Python 2
+        return codecs.open(filename, mode=mode, encoding=encoding)
+
+
+def detect_encoding(filename):
+    """Return file encoding."""
+    try:
+        # Python 3
+        try:
+            with open(filename, 'rb') as input_file:
+                encoding = tokenize.detect_encoding(input_file.readline)[0]
+
+                # Check for correctness of encoding
+                import io
+                with io.TextIOWrapper(input_file, encoding) as wrapper:
+                    wrapper.read()
+
+            return encoding
+        except (SyntaxError, LookupError, UnicodeDecodeError):
+            return 'latin-1'
+    except AttributeError:
+        # Python 2
+        encoding = 'utf-8'
+        try:
+            # Check for correctness of encoding
+            with open_with_encoding(filename, encoding) as input_file:
+                input_file.read()
+        except UnicodeDecodeError:
+            encoding = 'latin-1'
+
+        return encoding
+
+
+def read_from_filename(filename, readlines=False):
+    """Return contents of file."""
+    with open_with_encoding(filename,
+                            encoding=detect_encoding(filename)) as input_file:
+        return input_file.readlines() if readlines else input_file.read()
+
+
+class FixPEP8(object):
+
+    """Fix invalid code.
+
+    Fixer methods are prefixed "fix_". The _fix_source() method looks for these
+    automatically.
+
+    The fixer method can take either one or two arguments (in addition to
+    self). The first argument is "result", which is the error information from
+    pep8. The second argument, "logical", is required only for logical-line
+    fixes.
+
+    The fixer method can return the list of modified lines or None. An empty
+    list would mean that no changes were made. None would mean that only the
+    line reported in the pep8 error was modified. Note that the modified line
+    numbers that are returned are indexed at 1. This typically would correspond
+    with the line number reported in the pep8 error information.
+
+    [fixed method list]
+        - e111
+        - e121,e122,e123,e124,e125,e126,e127,e128
+        - e201,e202,e203
+        - e211
+        - e221,e222,e223,e224,e225
+        - e231
+        - e251
+        - e261,e262
+        - e271,e272,e273,e274
+        - e301,e302,e303
+        - e401
+        - e502
+        - e701,e702
+        - e711
+        - e721
+        - w291,w293
+        - w391
+        - w602,w603,w604
+
+    """
+
+    def __init__(self, filename, options, contents=None):
+        self.filename = filename
+        if contents is None:
+            self.source = read_from_filename(filename, readlines=True)
+        else:
+            sio = StringIO(contents)
+            self.source = sio.readlines()
+        self.original_source = copy.copy(self.source)
+        self.newline = find_newline(self.source)
+        self.options = options
+        self.indent_word = _get_indentword(''.join(self.source))
+        self.logical_start = None
+        self.logical_end = None
+        # method definition
+        self.fix_e111 = self.fix_e101
+        self.fix_e128 = self.fix_e127
+        self.fix_e202 = self.fix_e201
+        self.fix_e203 = self.fix_e201
+        self.fix_e211 = self.fix_e201
+        self.fix_e221 = self.fix_e271
+        self.fix_e222 = self.fix_e271
+        self.fix_e223 = self.fix_e271
+        self.fix_e241 = self.fix_e271
+        self.fix_e242 = self.fix_e224
+        self.fix_e261 = self.fix_e262
+        self.fix_e272 = self.fix_e271
+        self.fix_e273 = self.fix_e271
+        self.fix_e274 = self.fix_e271
+        self.fix_w191 = self.fix_e101
+
+    def _fix_source(self, results):
+        completed_lines = set()
+        for result in sorted(results, key=_priority_key):
+            if result['line'] in completed_lines:
+                continue
+
+            fixed_methodname = 'fix_%s' % result['id'].lower()
+            if hasattr(self, fixed_methodname):
+                fix = getattr(self, fixed_methodname)
+
+                is_logical_fix = len(inspect.getargspec(fix).args) > 2
+                if is_logical_fix:
+                    # Do not run logical fix if any lines have been modified.
+                    if completed_lines:
+                        continue
+
+                    logical = self._get_logical(result)
+                    if not logical:
+                        continue
+
+                    modified_lines = fix(result, logical)
+                else:
+                    modified_lines = fix(result)
+
+                if modified_lines:
+                    completed_lines.update(modified_lines)
+                elif modified_lines == []:  # Empty list means no fix
+                    if self.options.verbose >= 2:
+                        print(
+                            'Not fixing {f} on line {l}'.format(
+                                f=result['id'], l=result['line']),
+                            file=sys.stderr)
+                else:  # We assume one-line fix when None
+                    completed_lines.add(result['line'])
+            else:
+                if self.options.verbose >= 3:
+                    print("'%s' is not defined." % fixed_methodname,
+                          file=sys.stderr)
+                    info = result['info'].strip()
+                    print('%s:%s:%s:%s' % (self.filename,
+                                           result['line'],
+                                           result['column'],
+                                           info),
+                          file=sys.stderr)
+
+    def fix(self):
+        """Return a version of the source code with PEP 8 violations fixed."""
+        if pep8:
+            pep8_options = {
+                'ignore':
+                self.options.ignore and self.options.ignore.split(','),
+                'select':
+                self.options.select and self.options.select.split(','),
+            }
+            results = _execute_pep8(pep8_options, self.source)
+        else:
+            if self.options.verbose:
+                print('Running in compatibility mode. Consider '
+                      'upgrading to the latest pep8.',
+                      file=sys.stderr)
+            results = _spawn_pep8((['--ignore=' + self.options.ignore]
+                                   if self.options.ignore else []) +
+                                  (['--select=' + self.options.select]
+                                   if self.options.select else []) +
+                                  [self.filename])
+
+        if self.options.verbose:
+            print('{n} issues to fix'.format(
+                n=len(results)), file=sys.stderr)
+
+        self._fix_source(filter_results(source=''.join(self.source),
+                                        results=results))
+        return ''.join(self.source)
+
+    def fix_e101(self, _):
+        """Reindent all lines."""
+        reindenter = Reindenter(self.source, self.newline)
+        modified_line_numbers = reindenter.run()
+        if modified_line_numbers:
+            self.source = reindenter.fixed_lines()
+            return modified_line_numbers
+        else:
+            return []
+
+    def find_logical(self, force=False):
+        # make a variable which is the index of all the starts of lines
+        if not force and self.logical_start is not None:
+            return
+        logical_start = []
+        logical_end = []
+        last_newline = True
+        sio = StringIO(''.join(self.source))
+        parens = 0
+        for t in tokenize.generate_tokens(sio.readline):
+            if t[0] in [tokenize.COMMENT, tokenize.DEDENT,
+                        tokenize.INDENT, tokenize.NL,
+                        tokenize.ENDMARKER]:
+                continue
+            if not parens and t[0] in [
+                tokenize.NEWLINE, tokenize.SEMI
+            ]:
+                last_newline = True
+                logical_end.append((t[3][0] - 1, t[2][1]))
+                continue
+            if last_newline and not parens:
+                logical_start.append((t[2][0] - 1, t[2][1]))
+                last_newline = False
+            if t[0] == tokenize.OP:
+                if t[1] in '([{':
+                    parens += 1
+                elif t[1] in '}])':
+                    parens -= 1
+        self.logical_start = logical_start
+        self.logical_end = logical_end
+
+    def _get_logical(self, result):
+        """Return the logical line corresponding to the result.
+
+        Assumes input is already E702-clean.
+
+        """
+        try:
+            self.find_logical()
+        except (IndentationError, tokenize.TokenError):
+            return None
+
+        row = result['line'] - 1
+        col = result['column'] - 1
+        ls = None
+        le = None
+        for i in range(0, len(self.logical_start), 1):
+            x = self.logical_end[i]
+            if x[0] > row or (x[0] == row and x[1] > col):
+                le = x
+                ls = self.logical_start[i]
+                break
+        if ls is None:
+            return None
+        original = self.source[ls[0]:le[0] + 1]
+        return ls, le, original
+
+    def _fix_reindent(self, result, logical, fix_distinct=False):
+        """Fix a badly indented line.
+
+        This is done by adding or removing from its initial indent only.
+
+        """
+        if not logical:
+            return []
+        ls, _, original = logical
+        try:
+            rewrapper = Wrapper(original, hard_wrap=MAX_LINE_WIDTH)
+        except (tokenize.TokenError, IndentationError):
+            return []
+        valid_indents = rewrapper.pep8_expected()
+        if not rewrapper.rel_indent:
+            return []
+        if result['line'] > ls[0]:
+            # got a valid continuation line number from pep8
+            row = result['line'] - ls[0] - 1
+            # always pick the first option for this
+            valid = valid_indents[row]
+            got = rewrapper.rel_indent[row]
+        else:
+            # Line number from pep8 isn't a continuation line. Instead,
+            # compare our own function's result, look for the first mismatch,
+            # and just hope that we take fewer than 100 iterations to finish.
+            for row in range(0, len(original), 1):
+                valid = valid_indents[row]
+                got = rewrapper.rel_indent[row]
+                if valid != got:
+                    break
+        line = ls[0] + row
+        # always pick the expected indent, for now.
+        indent_to = valid[0]
+        if fix_distinct and indent_to == 4:
+            if len(valid) == 1:
+                return []
+            else:
+                indent_to = valid[1]
+
+        if got != indent_to:
+            orig_line = self.source[line]
+            new_line = ' ' * (indent_to) + orig_line.lstrip()
+            if new_line == orig_line:
+                return []
+            else:
+                self.source[line] = new_line
+                return [line + 1]  # Line indexed at 1
+        else:
+            return []
+
+    def fix_e121(self, result, logical):
+        """Fix indentation to be a multiple of four."""
+        # Fix by adjusting initial indent level.
+        return self._fix_reindent(result, logical)
+
+    def fix_e122(self, result, logical):
+        """Add absent indentation for hanging indentation."""
+        # Fix by adding an initial indent.
+        return self._fix_reindent(result, logical)
+
+    def fix_e123(self, result, logical):
+        """Align closing bracket to match opening bracket."""
+        # Fix by deleting whitespace to the correct level.
+        if not logical:
+            return []
+        logical_lines = logical[2]
+        line_index = result['line'] - 1
+        original_line = self.source[line_index]
+
+        fixed_line = (_get_indentation(logical_lines[0]) +
+                      original_line.lstrip())
+        if fixed_line == original_line:
+            # Fall back to slower method.
+            return self._fix_reindent(result, logical)
+        else:
+            self.source[line_index] = fixed_line
+
+    def fix_e124(self, result, logical):
+        """Align closing bracket to match visual indentation."""
+        # Fix by inserting whitespace before the closing bracket.
+        return self._fix_reindent(result, logical)
+
+    def fix_e125(self, result, logical):
+        """Indent to distinguish line from next logical line."""
+        # Fix by indenting the line in error to the next stop.
+        modified_lines = self._fix_reindent(result, logical, fix_distinct=True)
+        if modified_lines:
+            return modified_lines
+        else:
+            # Fallback
+            line_index = result['line'] - 1
+            original_line = self.source[line_index]
+            self.source[line_index] = self.indent_word + original_line
+
+    def fix_e126(self, result, logical):
+        """Fix over-indented hanging indentation."""
+        # fix by deleting whitespace to the left
+        if not logical:
+            return []
+        logical_lines = logical[2]
+        line_index = result['line'] - 1
+        original = self.source[line_index]
+
+        fixed = (_get_indentation(logical_lines[0]) +
+                 self.indent_word + original.lstrip())
+        if fixed == original:
+            # Fallback to slower method.
+            return self._fix_reindent(result, logical)
+        else:
+            self.source[line_index] = fixed
+
+    def fix_e127(self, result, logical):
+        """Fix visual indentation."""
+        # Fix by inserting/deleting whitespace to the correct level.
+        modified_lines = self._align_visual_indent(result, logical)
+        if modified_lines:
+            return modified_lines
+        else:
+            # Fallback to slower method.
+            return self._fix_reindent(result, logical)
+
+    def _align_visual_indent(self, result, logical):
+        """Correct visual indent.
+
+        This includes over (E127) and under (E128) indented lines.
+
+        """
+        if not logical:
+            return []
+        logical_lines = logical[2]
+        line_index = result['line'] - 1
+        original = self.source[line_index]
+        fixed = original
+
+        if '(' in logical_lines[0]:
+            fixed = logical_lines[0].find('(') * ' ' + original.lstrip()
+        elif logical_lines[0].rstrip().endswith('\\'):
+            fixed = (_get_indentation(logical_lines[0]) +
+                     self.indent_word + original.lstrip())
+        else:
+            return []
+
+        if fixed == original:
+            return []
+        else:
+            self.source[line_index] = fixed
+
+    def fix_e201(self, result):
+        """Remove extraneous whitespace."""
+        line_index = result['line'] - 1
+        target = self.source[line_index]
+        offset = result['column'] - 1
+
+        # When multiline strings are involved, pep8 reports the error as
+        # being at the start of the multiline string, which doesn't work
+        # for us.
+        if '"""' in target or "'''" in target:
+            return []
+
+        fixed = fix_whitespace(target,
+                               offset=offset,
+                               replacement='')
+
+        if fixed == target:
+            return []
+        else:
+            self.source[line_index] = fixed
+
+    def fix_e224(self, result):
+        """Remove extraneous whitespace around operator."""
+        target = self.source[result['line'] - 1]
+        offset = result['column'] - 1
+        fixed = target[:offset] + target[offset:].replace('\t', ' ')
+        self.source[result['line'] - 1] = fixed
+
+    def fix_e225(self, result):
+        """Fix missing whitespace around operator."""
+        target = self.source[result['line'] - 1]
+        offset = result['column'] - 1
+        fixed = target[:offset] + ' ' + target[offset:]
+
+        # Only proceed if non-whitespace characters match.
+        # And make sure we don't break the indentation.
+        if (fixed.replace(' ', '') == target.replace(' ', '') and
+                _get_indentation(fixed) == _get_indentation(target)):
+            self.source[result['line'] - 1] = fixed
+        else:
+            return []
+
+    def fix_e231(self, result):
+        """Add missing whitespace."""
+        line_index = result['line'] - 1
+        target = self.source[line_index]
+        offset = result['column']
+        fixed = target[:offset] + ' ' + target[offset:]
+        self.source[line_index] = fixed
+
+    def fix_e251(self, result):
+        """Remove whitespace around parameter '=' sign."""
+        line_index = result['line'] - 1
+        target = self.source[line_index]
+
+        # This is necessary since pep8 sometimes reports columns that goes
+        # past the end of the physical line. This happens in cases like,
+        # foo(bar\n=None)
+        c = min(result['column'] - 1,
+                len(target) - 1)
+
+        if target[c].strip():
+            fixed = target
+        else:
+            fixed = target[:c].rstrip() + target[c:].lstrip()
+
+        # There could be an escaped newline
+        #
+        #     def foo(a=\
+        #             1)
+        if (fixed.endswith('=\\\n') or
+                fixed.endswith('=\\\r\n') or
+                fixed.endswith('=\\\r')):
+            self.source[line_index] = fixed.rstrip('\n\r \t\\')
+            self.source[line_index + 1] = \
+                self.source[line_index + 1].lstrip()
+            return [line_index + 1, line_index + 2]  # Line indexed at 1
+
+        self.source[result['line'] - 1] = fixed
+
+    def fix_e262(self, result):
+        """Fix spacing after comment hash."""
+        target = self.source[result['line'] - 1]
+        offset = result['column']
+
+        code = target[:offset].rstrip(' \t#')
+        comment = target[offset:].lstrip(' \t#')
+
+        fixed = code + ('  # ' + comment if comment.strip()
+                        else self.newline)
+
+        self.source[result['line'] - 1] = fixed
+
+    def fix_e271(self, result):
+        """Fix extraneous whitespace around keywords."""
+        line_index = result['line'] - 1
+        target = self.source[line_index]
+        offset = result['column'] - 1
+
+        fixed = fix_whitespace(target,
+                               offset=offset,
+                               replacement=' ')
+
+        if fixed == target:
+            return []
+        else:
+            self.source[line_index] = fixed
+
+    def fix_e301(self, result):
+        """Add missing blank line."""
+        cr = self.newline
+        self.source[result['line'] - 1] = cr + self.source[result['line'] - 1]
+
+    def fix_e302(self, result):
+        """Add missing 2 blank lines."""
+        add_linenum = 2 - int(result['info'].split()[-1])
+        cr = self.newline * add_linenum
+        self.source[result['line'] - 1] = cr + self.source[result['line'] - 1]
+
+    def fix_e303(self, result):
+        """Remove extra blank lines."""
+        delete_linenum = int(result['info'].split('(')[1].split(')')[0]) - 2
+        delete_linenum = max(1, delete_linenum)
+
+        # We need to count because pep8 reports an offset line number if there
+        # are comments.
+        cnt = 0
+        line = result['line'] - 2
+        modified_lines = []
+        while cnt < delete_linenum:
+            if line < 0:
+                break
+            if not self.source[line].strip():
+                self.source[line] = ''
+                modified_lines.append(1 + line)  # Line indexed at 1
+                cnt += 1
+            line -= 1
+
+        return modified_lines
+
+    def fix_e304(self, result):
+        """Remove blank line following function decorator."""
+        line = result['line'] - 2
+        if not self.source[line].strip():
+            self.source[line] = ''
+
+    def fix_e401(self, result):
+        """Put imports on separate lines."""
+        line_index = result['line'] - 1
+        target = self.source[line_index]
+        offset = result['column'] - 1
+
+        if not target.lstrip().startswith('import'):
+            return []
+
+        # pep8 (1.3.1) reports false positive if there is an import statement
+        # followed by a semicolon and some unrelated statement with commas in
+        # it.
+        if ';' in target:
+            return []
+
+        indentation = target.split('import ')[0]
+        fixed = (target[:offset].rstrip('\t ,') + self.newline +
+                 indentation + 'import ' + target[offset:].lstrip('\t ,'))
+        self.source[line_index] = fixed
+
+    def fix_e501(self, result):
+        """Try to make lines fit within 79 characters."""
+        line_index = result['line'] - 1
+        target = self.source[line_index]
+
+        indent = _get_indentation(target)
+        source = target[len(indent):]
+        sio = StringIO(target)
+
+        # Check for multiline string.
+        try:
+            tokens = list(tokenize.generate_tokens(sio.readline))
+        except (tokenize.TokenError, IndentationError):
+            multi_line_candidate = break_multi_line(
+                target, newline=self.newline, indent_word=self.indent_word)
+
+            if multi_line_candidate:
+                self.source[line_index] = multi_line_candidate
+                return
+            else:
+                return []
+
+        # Prefer
+        # my_long_function_name(
+        #     x, y, z, ...)
+        #
+        # over
+        # my_long_function_name(x, y,
+        #     z, ...)
+        candidate0 = _shorten_line(tokens, source, target, indent,
+                                   self.indent_word, newline=self.newline,
+                                   reverse=False)
+        candidate1 = _shorten_line(tokens, source, target, indent,
+                                   self.indent_word, newline=self.newline,
+                                   reverse=True)
+        if candidate0 and candidate1:
+            if candidate0.split(self.newline)[0].endswith('('):
+                self.source[line_index] = candidate0
+            else:
+                self.source[line_index] = candidate1
+        elif candidate0:
+            self.source[line_index] = candidate0
+        elif candidate1:
+            self.source[line_index] = candidate1
+        else:
+            # Otherwise both don't work
+            return []
+
+    def fix_e502(self, result):
+        """Remove extraneous escape of newline."""
+        line_index = result['line'] - 1
+        target = self.source[line_index]
+        self.source[line_index] = target.rstrip('\n\r \t\\') + self.newline
+
+    def fix_e701(self, result):
+        """Put colon-separated compound statement on separate lines."""
+        line_index = result['line'] - 1
+        target = self.source[line_index]
+        c = result['column']
+
+        fixed_source = (target[:c] + self.newline +
+                        _get_indentation(target) + self.indent_word +
+                        target[c:].lstrip('\n\r \t\\'))
+        self.source[result['line'] - 1] = fixed_source
+
+    def fix_e702(self, result, logical):
+        """Put semicolon-separated compound statement on separate lines."""
+        logical_lines = logical[2]
+
+        line_index = result['line'] - 1
+        target = self.source[line_index]
+
+        if target.rstrip().endswith('\\'):
+            # Normalize '1; \\\n2' into '1; 2'.
+            self.source[line_index] = target.rstrip('\n \r\t\\')
+            self.source[line_index + 1] = self.source[line_index + 1].lstrip()
+            return [line_index + 1, line_index + 2]
+
+        if target.rstrip().endswith(';'):
+            self.source[line_index] = target.rstrip('\n \r\t;') + self.newline
+            return
+
+        offset = result['column'] - 1
+        first = target[:offset].rstrip(';').rstrip()
+        second = (_get_indentation(logical_lines[0]) +
+                  target[offset:].lstrip(';').lstrip())
+
+        self.source[line_index] = first + self.newline + second
+
+    def fix_e711(self, result):
+        """Fix comparison."""
+        line_index = result['line'] - 1
+        target = self.source[line_index]
+        offset = result['column'] - 1
+
+        right_offset = offset + 2
+        if right_offset >= len(target):
+            return []
+
+        left = target[:offset].rstrip()
+        center = target[offset:right_offset]
+        right = target[right_offset:].lstrip()
+
+        if not right.startswith('None'):
+            return []
+
+        if center.strip() == '==':
+            new_center = 'is'
+        elif center.strip() == '!=':
+            new_center = 'is not'
+        else:
+            return []
+
+        self.source[line_index] = ' '.join([left, new_center, right])
+
+    def fix_e721(self, _):
+        """Switch to use isinstance()."""
+        return self.refactor('idioms')
+
+    def fix_w291(self, result):
+        """Remove trailing whitespace."""
+        fixed_line = self.source[result['line'] - 1].rstrip()
+        self.source[result['line'] - 1] = '%s%s' % (fixed_line, self.newline)
+
+    def fix_w293(self, result):
+        """Remove trailing whitespace on blank line."""
+        assert not self.source[result['line'] - 1].strip()
+        self.source[result['line'] - 1] = self.newline
+
+    def fix_w391(self, _):
+        """Remove trailing blank lines."""
+        blank_count = 0
+        for line in reversed(self.source):
+            line = line.rstrip()
+            if line:
+                break
+            else:
+                blank_count += 1
+
+        original_length = len(self.source)
+        self.source = self.source[:original_length - blank_count]
+        return range(1, 1 + original_length)
+
+    def refactor(self, fixer_name, ignore=None):
+        """Return refactored code using lib2to3.
+
+        Skip if ignore string is produced in the refactored code.
+
+        """
+        from lib2to3 import pgen2
+        try:
+            new_text = refactor_with_2to3(''.join(self.source),
+                                          fixer_name=fixer_name)
+        except (pgen2.parse.ParseError,
+                UnicodeDecodeError, UnicodeEncodeError):
+            return []
+
+        try:
+            original = unicode(''.join(self.source).strip(), 'utf-8')
+        except (NameError, TypeError):
+            original = ''.join(self.source).strip()
+        if original == new_text.strip():
+            return []
+        else:
+            if ignore:
+                if ignore in new_text and ignore not in ''.join(self.source):
+                    return []
+            original_length = len(self.source)
+            self.source = [new_text]
+            return range(1, 1 + original_length)
+
+    def fix_w601(self, _):
+        """Replace the {}.has_key() form with 'in'."""
+        return self.refactor('has_key')
+
+    def fix_w602(self, _):
+        """Fix deprecated form of raising exception."""
+        return self.refactor('raise',
+                             ignore='with_traceback')
+
+    def fix_w603(self, _):
+        """Replace <> with !=."""
+        return self.refactor('ne')
+
+    def fix_w604(self, _):
+        """Replace backticks with repr()."""
+        return self.refactor('repr')
+
+
+def find_newline(source):
+    """Return type of newline used in source."""
+    cr, lf, crlf = 0, 0, 0
+    for s in source:
+        if CRLF in s:
+            crlf += 1
+        elif CR in s:
+            cr += 1
+        elif LF in s:
+            lf += 1
+    _max = max(cr, crlf, lf)
+    if _max == lf:
+        return LF
+    elif _max == crlf:
+        return CRLF
+    elif _max == cr:
+        return CR
+    else:
+        return LF
+
+
+def _get_indentword(source):
+    """Return indentation type."""
+    sio = StringIO(source)
+    indent_word = '    '  # Default in case source has no indentation
+    try:
+        for t in tokenize.generate_tokens(sio.readline):
+            if t[0] == token.INDENT:
+                indent_word = t[1]
+                break
+    except (tokenize.TokenError, IndentationError):
+        pass
+    return indent_word
+
+
+def _get_indentation(line):
+    """Return leading whitespace."""
+    if line.strip():
+        non_whitespace_index = len(line) - len(line.lstrip())
+        return line[:non_whitespace_index]
+    else:
+        return ''
+
+
+def _analyze_pep8result(result):
+    tmp = result.split(':')
+    filename = tmp[0]
+    line = int(tmp[1])
+    column = int(tmp[2])
+    info = ' '.join(result.split()[1:])
+    pep8id = info.lstrip().split()[0]
+    return dict(id=pep8id, filename=filename, line=line,
+                column=column, info=info)
+
+
+def _get_difftext(old, new, filename):
+    diff = unified_diff(old, new, 'original/' + filename, 'fixed/' + filename)
+    return ''.join(diff)
+
+
+def _priority_key(pep8_result):
+    """Key for sorting PEP8 results.
+
+    Global fixes should be done first. This is important for things
+    like indentation.
+
+    """
+    priority = ['e101', 'e111', 'w191',  # Global fixes
+                'e701',  # Fix multiline colon-based before semicolon based
+                'e702',  # Break multiline statements early
+                'e225', 'e231',  # things that make lines longer
+                'e201',  # Remove extraneous whitespace before breaking lines
+                'e501',  # before we break lines
+                ]
+    key = pep8_result['id'].lower()
+    if key in priority:
+        return priority.index(key)
+    else:
+        # Lowest priority
+        return len(priority)
+
+
+def _shorten_line(tokens, source, target, indentation, indent_word, newline,
+                  reverse=False):
+    """Separate line at OPERATOR."""
+    max_line_width_minus_indentation = MAX_LINE_WIDTH - len(indentation)
+    if reverse:
+        tokens.reverse()
+    for tkn in tokens:
+        # Don't break on '=' after keyword as this violates PEP 8.
+        if token.OP == tkn[0] and tkn[1] != '=':
+            offset = tkn[2][1] + 1
+            if reverse:
+                if offset > (max_line_width_minus_indentation -
+                             len(indent_word)):
+                    continue
+            else:
+                if (len(target.rstrip()) - offset >
+                        (max_line_width_minus_indentation -
+                         len(indent_word))):
+                    continue
+            first = source[:offset - len(indentation)]
+
+            second_indent = indentation
+            if first.rstrip().endswith('('):
+                second_indent += indent_word
+            elif '(' in first:
+                second_indent += ' ' * (1 + first.find('('))
+            else:
+                second_indent += indent_word
+
+            second = (second_indent +
+                      source[offset - len(indentation):].lstrip())
+            if not second.strip():
+                continue
+
+            # Don't modify if lines are not short enough
+            if len(first) > max_line_width_minus_indentation:
+                continue
+            if len(second) > MAX_LINE_WIDTH:  # Already includes indentation
+                continue
+            # Do not begin a line with a comma
+            if second.lstrip().startswith(','):
+                continue
+            # Do end a line with a dot
+            if first.rstrip().endswith('.'):
+                continue
+            if tkn[1] in '+-*/':
+                fixed = first + ' \\' + newline + second
+            else:
+                fixed = first + newline + second
+            if check_syntax(fixed):
+                return indentation + fixed
+    return None
+
+
+def fix_whitespace(line, offset, replacement):
+    """Replace whitespace at offset and return fixed line."""
+    # Replace escaped newlines too
+    left = line[:offset].rstrip('\n\r \t\\')
+    right = line[offset:].lstrip('\n\r \t\\')
+    if right.startswith('#'):
+        return line
+    else:
+        return left + replacement + right
+
+
+def _spawn_pep8(pep8_options):
+    """Execute pep8 via subprocess.Popen."""
+    for path in os.environ['PATH'].split(':'):
+        if os.path.exists(os.path.join(path, PEP8_BIN)):
+            cmd = ([os.path.join(path, PEP8_BIN)] +
+                   pep8_options)
+            p = Popen(cmd, stdout=PIPE)
+            output = p.communicate()[0].decode('utf-8')
+            return [_analyze_pep8result(l)
+                    for l in output.splitlines()]
+    raise Exception("'%s' is not found." % PEP8_BIN)
+
+
+def _execute_pep8(pep8_options, source):
+    """Execute pep8 via python method calls."""
+    class QuietReport(pep8.BaseReport):
+
+        """Version of checker that does not print."""
+
+        def __init__(self, options):
+            super(QuietReport, self).__init__(options)
+            self.__full_error_results = []
+
+        def error(self, line_number, offset, text, _):
+            """Collect errors."""
+            code = super(QuietReport, self).error(line_number, offset, text, _)
+            if code:
+                self.__full_error_results.append(
+                    dict(id=code, line=line_number,
+                         column=offset + 1, info=text))
+
+        def full_error_results(self):
+            """Return error results in detail.
+
+            Results are in the form of a list of dictionaries. Each dictionary
+            contains 'id', 'line', 'column', and 'info'.
+
+            """
+            return self.__full_error_results
+
+    checker = pep8.Checker('', lines=source,
+                           reporter=QuietReport, **pep8_options)
+    checker.check_all()
+    return checker.report.full_error_results()
+
+
+class Reindenter(object):
+
+    """Reindents badly-indented code to uniformly use four-space indentation.
+
+    Released to the public domain, by Tim Peters, 03 October 2000.
+
+    """
+
+    def __init__(self, input_text, newline):
+        self.newline = newline
+
+        self.find_stmt = 1  # next token begins a fresh stmt?
+        self.level = 0  # current indent level
+
+        # Raw file lines.
+        self.raw = input_text
+        self.after = None
+
+        self.string_content_line_numbers = multiline_string_lines(
+            ''.join(self.raw))
+
+        # File lines, rstripped & tab-expanded.  Dummy at start is so
+        # that we can use tokenize's 1-based line numbering easily.
+        # Note that a line is all-blank iff it is a newline.
+        self.lines = []
+        for line_number, line in enumerate(self.raw, start=1):
+            # Do not modify if inside a multi-line string.
+            if line_number in self.string_content_line_numbers:
+                self.lines.append(line)
+            else:
+                # Only expand leading tabs.
+                self.lines.append(_get_indentation(line).expandtabs() +
+                                  line.strip() + newline)
+
+        self.lines.insert(0, None)
+        self.index = 1  # index into self.lines of next line
+
+        # List of (lineno, indentlevel) pairs, one for each stmt and
+        # comment line.  indentlevel is -1 for comment lines, as a
+        # signal that tokenize doesn't know what to do about them;
+        # indeed, they're our headache!
+        self.stats = []
+
+    def run(self):
+        """Fix indentation and return modified line numbers.
+
+        Line numbers are indexed at 1.
+
+        """
+        tokens = tokenize.generate_tokens(self.getline)
+        try:
+            for t in tokens:
+                self.tokeneater(*t)
+        except (tokenize.TokenError, IndentationError):
+            return set()
+        # Remove trailing empty lines.
+        lines = self.lines
+        while lines and lines[-1] == self.newline:
+            lines.pop()
+        # Sentinel.
+        stats = self.stats
+        stats.append((len(lines), 0))
+        # Map count of leading spaces to # we want.
+        have2want = {}
+        # Program after transformation.
+        after = self.after = []
+        # Copy over initial empty lines -- there's nothing to do until
+        # we see a line with *something* on it.
+        i = stats[0][0]
+        after.extend(lines[1:i])
+        for i in range(len(stats) - 1):
+            thisstmt, thislevel = stats[i]
+            nextstmt = stats[i + 1][0]
+            have = _leading_space_count(lines[thisstmt])
+            want = thislevel * 4
+            if want < 0:
+                # A comment line.
+                if have:
+                    # An indented comment line.  If we saw the same
+                    # indentation before, reuse what it most recently
+                    # mapped to.
+                    want = have2want.get(have, - 1)
+                    if want < 0:
+                        # Then it probably belongs to the next real stmt.
+                        for j in range(i + 1, len(stats) - 1):
+                            jline, jlevel = stats[j]
+                            if jlevel >= 0:
+                                if have == _leading_space_count(lines[jline]):
+                                    want = jlevel * 4
+                                break
+                    if want < 0:           # Maybe it's a hanging
+                                           # comment like this one,
+                        # in which case we should shift it like its base
+                        # line got shifted.
+                        for j in range(i - 1, -1, -1):
+                            jline, jlevel = stats[j]
+                            if jlevel >= 0:
+                                want = (have + _leading_space_count(
+                                        after[jline - 1]) -
+                                        _leading_space_count(lines[jline]))
+                                break
+                    if want < 0:
+                        # Still no luck -- leave it alone.
+                        want = have
+                else:
+                    want = 0
+            assert want >= 0
+            have2want[have] = want
+            diff = want - have
+            if diff == 0 or have == 0:
+                after.extend(lines[thisstmt:nextstmt])
+            else:
+                for line_number, line in enumerate(lines[thisstmt:nextstmt],
+                                                   start=thisstmt):
+                    if line_number in self.string_content_line_numbers:
+                        after.append(line)
+                    elif diff > 0:
+                        if line == self.newline:
+                            after.append(line)
+                        else:
+                            after.append(' ' * diff + line)
+                    else:
+                        remove = min(_leading_space_count(line), -diff)
+                        after.append(line[remove:])
+
+        if self.raw == self.after:
+            return set()
+        else:
+            return (set(range(1, 1 + len(self.raw))) -
+                    self.string_content_line_numbers)
+
+    def fixed_lines(self):
+        return self.after
+
+    def getline(self):
+        """Line-getter for tokenize."""
+        if self.index >= len(self.lines):
+            line = ""
+        else:
+            line = self.lines[self.index]
+            self.index += 1
+        return line
+
+    def tokeneater(self, token_type, _, start, __, line):
+        """Line-eater for tokenize."""
+        sline = start[0]
+        if token_type == tokenize.NEWLINE:
+            # A program statement, or ENDMARKER, will eventually follow,
+            # after some (possibly empty) run of tokens of the form
+            #     (NL | COMMENT)* (INDENT | DEDENT+)?
+            self.find_stmt = 1
+
+        elif token_type == tokenize.INDENT:
+            self.find_stmt = 1
+            self.level += 1
+
+        elif token_type == tokenize.DEDENT:
+            self.find_stmt = 1
+            self.level -= 1
+
+        elif token_type == tokenize.COMMENT:
+            if self.find_stmt:
+                self.stats.append((sline, -1))
+                # but we're still looking for a new stmt, so leave
+                # find_stmt alone
+
+        elif token_type == tokenize.NL:
+            pass
+
+        elif self.find_stmt:
+            # This is the first "real token" following a NEWLINE, so it
+            # must be the first token of the next program statement, or an
+            # ENDMARKER.
+            self.find_stmt = 0
+            if line:   # not endmarker
+                self.stats.append((sline, self.level))
+
+
+class Wrapper(object):
+
+    """Class for functions relating to continuation lines and line folding.
+
+    Each instance operates on a single logical line.
+
+    """
+
+    SKIP_TOKENS = frozenset([
+        tokenize.COMMENT, tokenize.NL, tokenize.INDENT,
+        tokenize.DEDENT, tokenize.NEWLINE, tokenize.ENDMARKER
+    ])
+
+    def __init__(self, physical_lines, hard_wrap=79, soft_wrap=72):
+        if type(physical_lines) != list:
+            physical_lines = physical_lines.splitlines(keepends=True)
+        self.lines = physical_lines
+        self.index = 0
+        self.hard_wrap = hard_wrap
+        self.soft_wrap = soft_wrap
+        self.tokens = list()
+        self.rel_indent = None
+        sio = StringIO(''.join(physical_lines))
+        for t in tokenize.generate_tokens(sio.readline):
+            if not len(self.tokens) and t[0] in self.SKIP_TOKENS:
+                continue
+            if t[0] != tokenize.ENDMARKER:
+                self.tokens.append(t)
+
+        self.logical_line = self.build_tokens_logical(self.tokens)
+
+    def build_tokens_logical(self, tokens):
+        """Build a logical line from a list of tokens.
+
+        Return the logical line and a list of (offset, token) tuples. Does
+        not mute strings like the version in pep8.py.
+
+        """
+        # from pep8.py with minor modifications
+        logical = []
+        previous = None
+        for t in tokens:
+            token_type, text = t[0:2]
+            if token_type in self.SKIP_TOKENS:
+                continue
+            if previous:
+                end_line, end = previous[3]
+                start_line, start = t[2]
+                if end_line != start_line:  # different row
+                    prev_text = self.lines[end_line - 1][end - 1]
+                    if prev_text == ',' or (prev_text not in '{[('
+                                            and text not in '}])'):
+                        logical.append(' ')
+                elif end != start:  # different column
+                    fill = self.lines[end_line - 1][end:start]
+                    logical.append(fill)
+            logical.append(text)
+            previous = t
+        logical_line = ''.join(logical)
+        assert logical_line.lstrip() == logical_line
+        assert logical_line.rstrip() == logical_line
+        return logical_line
+
+    def pep8_expected(self):
+        """Replicate logic in pep8.py, to know what level to indent things to.
+
+        Return a list of lists; each list represents valid indent levels for
+        the line in question, relative from the initial indent.  However, the
+        first entry is the indent level which was expected.
+
+        """
+        # What follows is an adjusted version of
+        # pep8.py:continuation_line_indentation. All of the comments have been
+        # stripped and the 'yield' statements replaced with 'pass'.
+        tokens = self.tokens
+        if not tokens:
+            return
+
+        first_row = tokens[0][2][0]
+        nrows = 1 + tokens[-1][2][0] - first_row
+
+        # here are the return values
+        valid_indents = [list()] * nrows
+        indent_level = tokens[0][2][1]
+        valid_indents[0].append(indent_level)
+
+        if nrows == 1:
+            # bug, really.
+            return valid_indents
+
+        indent_next = self.logical_line.endswith(':')
+
+        row = depth = 0
+        parens = [0] * nrows
+        self.rel_indent = rel_indent = [0] * nrows
+        indent = [indent_level]
+        indent_chances = {}
+        last_indent = (0, 0)
+        last_token_multiline = None
+
+        for token_type, text, start, end, _ in self.tokens:
+            newline = row < start[0] - first_row
+            if newline:
+                row = start[0] - first_row
+                newline = (not last_token_multiline and
+                           token_type not in (tokenize.NL, tokenize.NEWLINE))
+
+            if newline:
+                # This is where the differences start. Instead of looking at
+                # the line and determining whether the observed indent matches
+                # our expectations, we decide which type of indentation is in
+                # use at the given indent level, and return the offset. This
+                # algorithm is susceptible to "carried errors", but should
+                # through repeated runs eventually solve indentation for
+                # multi-line expressions less than PEP8_PASSES_MAX lines long.
+
+                if depth:
+                    for open_row in range(row - 1, -1, -1):
+                        if parens[open_row]:
+                            break
+                else:
+                    open_row = 0
+
+                # That's all we get to work with. This code attempts to
+                # "reverse" the below logic, and place into the valid indents
+                # list
+                vi = []
+                add_second_chances = False
+                if token_type == tokenize.OP and text in ']})':
+                    # this line starts with a closing bracket, so it needs to
+                    # be closed at the same indent as the opening one.
+                    if indent[depth]:
+                        # hanging indent
+                        vi.append(indent[depth])
+                    else:
+                        # visual indent
+                        vi.append(indent_level + rel_indent[open_row])
+                elif depth and indent[depth]:
+                    # visual indent was previously confirmed.
+                    vi.append(indent[depth])
+                    add_second_chances = True
+                elif depth and True in indent_chances.values():
+                    # visual indent happened before, so stick to
+                    # visual indent this time.
+                    if depth > 1 and indent[depth - 1]:
+                        vi.append(indent[depth - 1])
+                    else:
+                        # stupid fallback
+                        vi.append(indent_level + 4)
+                    add_second_chances = True
+                elif not depth:
+                    vi.append(indent_level + 4)
+                else:
+                    # must be in hanging indent
+                    hang = rel_indent[open_row] + 4
+                    vi.append(indent_level + hang)
+
+                # about the best we can do without look-ahead
+                if indent_next and vi[0] == indent_level + 4 and \
+                        nrows == row + 1:
+                    vi[0] += 4
+
+                if add_second_chances:
+                    # visual indenters like to line things up.
+                    min_indent = vi[0]
+                    for col, what in indent_chances.items():
+                        if col > min_indent and (
+                            what is True or
+                            (what == str and token_type == tokenize.STRING) or
+                            (what == text and token_type == tokenize.OP)
+                        ):
+                            vi.append(col)
+                    vi = sorted(vi)
+
+                valid_indents[row] = vi
+
+                # ...returning to original continuation_line_identation func...
+                visual_indent = indent_chances.get(start[1])
+                last_indent = start
+                rel_indent[row] = start[1] - indent_level
+                hang = rel_indent[row] - rel_indent[open_row]
+
+                if token_type == tokenize.OP and text in ']})':
+                    if indent[depth]:
+                        if start[1] != indent[depth]:
+                            pass  # E124
+                    elif hang:
+                        pass  # E123
+                elif visual_indent is True:
+                    if not indent[depth]:
+                        indent[depth] = start[1]
+                elif visual_indent in (text, str):
+                    pass
+                elif indent[depth] and start[1] < indent[depth]:
+                    pass  # E128
+                elif hang == 4 or (indent_next and rel_indent[row] == 8):
+                    pass
+                else:
+                    if hang <= 0:
+                        pass  # E122
+                    elif indent[depth]:
+                        pass  # E127
+                    elif hang % 4:
+                        pass  # E121
+                    else:
+                        pass  # E126
+
+            # line altered: comments shouldn't define a visual indent
+            if parens[row] and not indent[depth] and token_type not in (
+                tokenize.NL, tokenize.COMMENT
+            ):
+                indent[depth] = start[1]
+                indent_chances[start[1]] = True
+            elif token_type == tokenize.STRING or text in (
+                'u', 'ur', 'b', 'br'
+            ):
+                indent_chances[start[1]] = str
+
+            if token_type == tokenize.OP:
+                if text in '([{':
+                    depth += 1
+                    indent.append(0)
+                    parens[row] += 1
+                elif text in ')]}' and depth > 0:
+                    prev_indent = indent.pop() or last_indent[1]
+                    for d in range(depth):
+                        if indent[d] > prev_indent:
+                            indent[d] = 0
+                    for ind in list(indent_chances):
+                        if ind >= prev_indent:
+                            del indent_chances[ind]
+                    depth -= 1
+                    if depth and indent[depth]:  # modified
+                        indent_chances[indent[depth]] = True
+                    for idx in range(row, -1, -1):
+                        if parens[idx]:
+                            parens[idx] -= 1
+                            break
+                assert len(indent) == depth + 1
+                if start[1] not in indent_chances:
+                    indent_chances[start[1]] = text
+
+            last_token_multiline = (start[0] != end[0])
+
+        if indent_next and rel_indent[-1] == 4:
+            pass  # E125
+
+        return valid_indents
+
+
+def _leading_space_count(line):
+    """Return number of leading spaces in line."""
+    i = 0
+    while i < len(line) and line[i] == ' ':
+        i += 1
+    return i
+
+
+def refactor_with_2to3(source_text, fixer_name):
+    """Use lib2to3 to refactor the source.
+
+    Return the refactored source code.
+
+    """
+    from lib2to3 import refactor
+    fixers = ['lib2to3.fixes.fix_' + fixer_name]
+    tool = refactor.RefactoringTool(
+        fixer_names=fixers,
+        explicit=fixers)
+    try:
+        return unicode(tool.refactor_string(
+            source_text.decode('utf-8'), name=''))
+    except NameError:
+        return str(tool.refactor_string(source_text, name=''))
+
+
+def break_multi_line(source_text, newline, indent_word):
+    """Break first line of multi-line code.
+
+    Return None if a break is not possible.
+
+    """
+    # Handle special case only.
+    if ('(' in source_text and source_text.rstrip().endswith(',')):
+        index = 1 + source_text.find('(')
+        if index >= MAX_LINE_WIDTH:
+            return None
+
+        # Make sure we are not in a string.
+        for quote in ['"', "'"]:
+            if quote in source_text:
+                if source_text.find(quote) < index:
+                    return None
+
+        # Make sure we are not in a comment.
+        if '#' in source_text:
+            if source_text.find('#') < index:
+                return None
+
+        assert index < len(source_text)
+        return (
+            source_text[:index].rstrip() + newline +
+            _get_indentation(source_text) + indent_word +
+            source_text[index:].lstrip())
+    else:
+        return None
+
+
+def check_syntax(code):
+    """Return True if syntax is okay."""
+    try:
+        return compile(code, '<string>', 'exec')
+    except (SyntaxError, TypeError, UnicodeDecodeError):
+        return False
+
+
+def filter_results(source, results):
+    """Filter out spurious reports from pep8.
+
+    Currently we filter out errors about indentation in multiline strings.
+
+    """
+    string_line_numbers = multiline_string_lines(source)
+
+    for r in results:
+        if r['line'] in string_line_numbers:
+            if r['id'].lower().startswith('e1'):
+                continue
+            elif r['id'].lower() in ['e501', 'w191']:
+                continue
+
+        # Filter out incorrect E101 reports when there are no tabs.
+        # pep8 will complain about this even if the tab indentation found
+        # elsewhere is in a multi-line string.
+        if r['id'].lower() == 'e101' and '\t' not in source[r['line'] - 1]:
+            continue
+
+        yield r
+
+
+def multiline_string_lines(source):
+    """Return line numbers that are within multiline strings.
+
+    The line numbers are indexed at 1.
+
+    Docstrings are ignored.
+
+    """
+    sio = StringIO(source)
+    line_numbers = set()
+    previous_token_type = ''
+    try:
+        for t in tokenize.generate_tokens(sio.readline):
+            token_type = t[0]
+            token_string = t[1]
+            start_row = t[2][0]
+            end_row = t[3][0]
+
+            if (token_type == tokenize.STRING and
+                    starts_with_triple(token_string) and
+                    previous_token_type != tokenize.INDENT):
+                # We increment by one since we want the contents of the
+                # string.
+                line_numbers |= set(range(1 + start_row, 1 + end_row))
+
+            previous_token_type = token_type
+    except (IndentationError, tokenize.TokenError):
+        pass
+
+    return line_numbers
+
+
+def starts_with_triple(string):
+    """Return True if the string starts with triple single/double quotes."""
+    return (string.strip().startswith('"""') or
+            string.strip().startswith("'''"))
+
+
+def fix_file(filename, opts, output=sys.stdout):
+    tmp_source = read_from_filename(filename)
+
+    # Add missing newline (important for diff)
+    if tmp_source:
+        tmp_newline = find_newline(tmp_source)
+        if tmp_source == tmp_source.rstrip(tmp_newline):
+            tmp_source += tmp_newline
+
+    fix = FixPEP8(filename, opts, contents=tmp_source)
+    fixed_source = fix.fix()
+    original_source = copy.copy(fix.original_source)
+    tmp_filename = filename
+    if not pep8 or opts.in_place:
+        encoding = detect_encoding(filename)
+
+    for _ in range(opts.pep8_passes):
+        if fixed_source == tmp_source:
+            break
+        tmp_source = copy.copy(fixed_source)
+        if not pep8:
+            tmp_filename = tempfile.mkstemp()[1]
+            fp = open_with_encoding(tmp_filename, encoding=encoding, mode='w')
+            fp.write(fixed_source)
+            fp.close()
+        fix = FixPEP8(tmp_filename, opts, contents=tmp_source)
+        fixed_source = fix.fix()
+        if not pep8:
+            os.remove(tmp_filename)
+    del tmp_filename
+    del tmp_source
+
+    if opts.diff:
+        new = StringIO(''.join(fix.source))
+        new = new.readlines()
+        output.write(_get_difftext(original_source, new, filename))
+    elif opts.in_place:
+        fp = open_with_encoding(filename, encoding=encoding,
+                                mode='w')
+        fp.write(fixed_source)
+        fp.close()
+    else:
+        output.write(fixed_source)
+
+
+def parse_args(args):
+    """Parse command-line options."""
+    parser = OptionParser(usage='Usage: autopep8 [options] '
+                                '[filename [filename ...]]',
+                          version='autopep8: %s' % __version__,
+                          description=__doc__,
+                          prog='autopep8')
+    parser.add_option('-v', '--verbose', action='count', dest='verbose',
+                      default=0,
+                      help='print verbose messages; '
+                           'multiple -v result in more verbose messages')
+    parser.add_option('-d', '--diff', action='store_true', dest='diff',
+                      help='print the diff for the fixed source')
+    parser.add_option('-i', '--in-place', action='store_true',
+                      help='make changes to files in place')
+    parser.add_option('-r', '--recursive', action='store_true',
+                      help='run recursively; must be used with --in-place or '
+                           '--diff')
+    parser.add_option('-p', '--pep8-passes',
+                      default=100, type='int',
+                      help='maximum number of additional pep8 passes'
+                           ' (default: %default)')
+    parser.add_option('--list-fixes', action='store_true',
+                      help='list codes for fixes; '
+                           'used by --ignore and --select')
+    parser.add_option('--ignore', default='',
+                      help='do not fix these errors/warnings (e.g. E4,W)')
+    parser.add_option('--select', default='',
+                      help='fix only these errors/warnings (e.g. E4,W)')
+    opts, args = parser.parse_args(args)
+
+    if not len(args) and not opts.list_fixes:
+        parser.error('incorrect number of arguments')
+
+    if len(args) > 1 and not (opts.in_place or opts.diff):
+        parser.error('autopep8 only takes one filename as argument '
+                     'unless the "--in-place" or "--diff" options are '
+                     'used')
+
+    if opts.recursive and not (opts.in_place or opts.diff):
+        parser.error('--recursive must be used with --in-place or --diff')
+
+    if opts.in_place and opts.diff:
+        parser.error('--in-place and --diff are mutually exclusive')
+
+    return opts, args
+
+
+def supported_fixes():
+    """Yield pep8 error codes that autopep8 fixes.
+
+    Each item we yield is a tuple of the code followed by its description.
+
+    """
+    instance = FixPEP8(filename=None, options=None, contents='')
+    for attribute in dir(instance):
+        code = re.match('fix_([ew][0-9][0-9][0-9])', attribute)
+        if code:
+            yield (code.group(1).upper(),
+                   re.sub(r'\s+', ' ',
+                          getattr(instance, attribute).__doc__))
+
+
+def main():
+    """Tool main."""
+    opts, args = parse_args(sys.argv[1:])
+
+    if opts.list_fixes:
+        for code, description in supported_fixes():
+            print('{code} - {description}'.format(
+                code=code, description=description))
+        return 0
+
+    if opts.in_place or opts.diff:
+        filenames = list(set(args))
+    else:
+        assert len(args) == 1
+        assert not opts.recursive
+        filenames = args[:1]
+
+    if sys.version_info[0] >= 3:
+        output = sys.stdout
+    else:
+        output = codecs.getwriter(locale.getpreferredencoding())(sys.stdout)
+
+    while filenames:
+        name = filenames.pop(0)
+        if opts.recursive and os.path.isdir(name):
+            for root, directories, children in os.walk(name):
+                filenames += [os.path.join(root, f) for f in children
+                              if f.endswith('.py') and
+                              not f.startswith('.')]
+                for d in directories:
+                    if d.startswith('.'):
+                        directories.remove(d)
+        else:
+            if opts.verbose:
+                print('[file:%s]' % name, file=sys.stderr)
+            try:
+                fix_file(name, opts, output)
+            except IOError as error:
+                print(str(error), file=sys.stderr)
+
+
+if __name__ == '__main__':
+    try:
+        sys.exit(main())
+    except KeyboardInterrupt:
+        sys.exit(1)

pylibs/logilab/astng/COPYING

@@ -0,0 +1,339 @@
+		    GNU GENERAL PUBLIC LICENSE
+		       Version 2, June 1991
+
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+			    Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+License is intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users.  This
+General Public License applies to most of the Free Software
+Foundation's software and to any other program whose authors commit to
+using it.  (Some other Free Software Foundation software is covered by
+the GNU Lesser General Public License instead.)  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+this service if you wish), that you receive source code or can get it
+if you want it, that you can change the software or use pieces of it
+in new free programs; and that you know you can do these things.
+
+  To protect your rights, we need to make restrictions that forbid
+anyone to deny you these rights or to ask you to surrender the rights.
+These restrictions translate to certain responsibilities for you if you
+distribute copies of the software, or if you modify it.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must give the recipients all the rights that
+you have.  You must make sure that they, too, receive or can get the
+source code.  And you must show them these terms so they know their
+rights.
+
+  We protect your rights with two steps: (1) copyright the software, and
+(2) offer you this license which gives you legal permission to copy,
+distribute and/or modify the software.
+
+  Also, for each author's protection and ours, we want to make certain
+that everyone understands that there is no warranty for this free
+software.  If the software is modified by someone else and passed on, we
+want its recipients to know that what they have is not the original, so
+that any problems introduced by others will not reflect on the original
+authors' reputations.
+
+  Finally, any free program is threatened constantly by software
+patents.  We wish to avoid the danger that redistributors of a free
+program will individually obtain patent licenses, in effect making the
+program proprietary.  To prevent this, we have made it clear that any
+patent must be licensed for everyone's free use or not licensed at all.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+		    GNU GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License.  The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language.  (Hereinafter, translation is included without limitation in
+the term "modification".)  Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+  1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+  2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) You must cause the modified files to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    b) You must cause any work that you distribute or publish, that in
+    whole or in part contains or is derived from the Program or any
+    part thereof, to be licensed as a whole at no charge to all third
+    parties under the terms of this License.
+
+    c) If the modified program normally reads commands interactively
+    when run, you must cause it, when started running for such
+    interactive use in the most ordinary way, to print or display an
+    announcement including an appropriate copyright notice and a
+    notice that there is no warranty (or else, saying that you provide
+    a warranty) and that users may redistribute the program under
+    these conditions, and telling the user how to view a copy of this
+    License.  (Exception: if the Program itself is interactive but
+    does not normally print such an announcement, your work based on
+    the Program is not required to print an announcement.)
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+    a) Accompany it with the complete corresponding machine-readable
+    source code, which must be distributed under the terms of Sections
+    1 and 2 above on a medium customarily used for software interchange; or,
+
+    b) Accompany it with a written offer, valid for at least three
+    years, to give any third party, for a charge no more than your
+    cost of physically performing source distribution, a complete
+    machine-readable copy of the corresponding source code, to be
+    distributed under the terms of Sections 1 and 2 above on a medium
+    customarily used for software interchange; or,
+
+    c) Accompany it with the information you received as to the offer
+    to distribute corresponding source code.  (This alternative is
+    allowed only for noncommercial distribution and only if you
+    received the program in object code or executable form with such
+    an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it.  For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable.  However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+  4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License.  Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+  5. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Program or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+  6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+  7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+  8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation.  If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+  10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission.  For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this.  Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+			    NO WARRANTY
+
+  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+		     END OF TERMS AND CONDITIONS
+
+	    How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+Also add information on how to contact you by electronic and paper mail.
+
+If the program is interactive, make it output a short notice like this
+when it starts in an interactive mode:
+
+    Gnomovision version 69, Copyright (C) year name of author
+    Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, the commands you use may
+be called something other than `show w' and `show c'; they could even be
+mouse-clicks or menu items--whatever suits your program.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the program, if
+necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the program
+  `Gnomovision' (which makes passes at compilers) written by James Hacker.
+
+  <signature of Ty Coon>, 1 April 1989
+  Ty Coon, President of Vice
+
+This General Public License does not permit incorporating your program into
+proprietary programs.  If your program is a subroutine library, you may
+consider it more useful to permit linking proprietary applications with the
+library.  If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.

pylibs/logilab/astng/COPYING.LESSER

@@ -0,0 +1,510 @@
+
+                  GNU LESSER GENERAL PUBLIC LICENSE
+                       Version 2.1, February 1999
+
+ Copyright (C) 1991, 1999 Free Software Foundation, Inc.
+	51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+[This is the first released version of the Lesser GPL.  It also counts
+ as the successor of the GNU Library Public License, version 2, hence
+ the version number 2.1.]
+
+                            Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+Licenses are intended to guarantee your freedom to share and change
+free software--to make sure the software is free for all its users.
+
+  This license, the Lesser General Public License, applies to some
+specially designated software packages--typically libraries--of the
+Free Software Foundation and other authors who decide to use it.  You
+can use it too, but we suggest you first think carefully about whether
+this license or the ordinary General Public License is the better
+strategy to use in any particular case, based on the explanations
+below.
+
+  When we speak of free software, we are referring to freedom of use,
+not price.  Our General Public Licenses are designed to make sure that
+you have the freedom to distribute copies of free software (and charge
+for this service if you wish); that you receive source code or can get
+it if you want it; that you can change the software and use pieces of
+it in new free programs; and that you are informed that you can do
+these things.
+
+  To protect your rights, we need to make restrictions that forbid
+distributors to deny you these rights or to ask you to surrender these
+rights.  These restrictions translate to certain responsibilities for
+you if you distribute copies of the library or if you modify it.
+
+  For example, if you distribute copies of the library, whether gratis
+or for a fee, you must give the recipients all the rights that we gave
+you.  You must make sure that they, too, receive or can get the source
+code.  If you link other code with the library, you must provide
+complete object files to the recipients, so that they can relink them
+with the library after making changes to the library and recompiling
+it.  And you must show them these terms so they know their rights.
+
+  We protect your rights with a two-step method: (1) we copyright the
+library, and (2) we offer you this license, which gives you legal
+permission to copy, distribute and/or modify the library.
+
+  To protect each distributor, we want to make it very clear that
+there is no warranty for the free library.  Also, if the library is
+modified by someone else and passed on, the recipients should know
+that what they have is not the original version, so that the original
+author's reputation will not be affected by problems that might be
+introduced by others.
+
+  Finally, software patents pose a constant threat to the existence of
+any free program.  We wish to make sure that a company cannot
+effectively restrict the users of a free program by obtaining a
+restrictive license from a patent holder.  Therefore, we insist that
+any patent license obtained for a version of the library must be
+consistent with the full freedom of use specified in this license.
+
+  Most GNU software, including some libraries, is covered by the
+ordinary GNU General Public License.  This license, the GNU Lesser
+General Public License, applies to certain designated libraries, and
+is quite different from the ordinary General Public License.  We use
+this license for certain libraries in order to permit linking those
+libraries into non-free programs.
+
+  When a program is linked with a library, whether statically or using
+a shared library, the combination of the two is legally speaking a
+combined work, a derivative of the original library.  The ordinary
+General Public License therefore permits such linking only if the
+entire combination fits its criteria of freedom.  The Lesser General
+Public License permits more lax criteria for linking other code with
+the library.
+
+  We call this license the "Lesser" General Public License because it
+does Less to protect the user's freedom than the ordinary General
+Public License.  It also provides other free software developers Less
+of an advantage over competing non-free programs.  These disadvantages
+are the reason we use the ordinary General Public License for many
+libraries.  However, the Lesser license provides advantages in certain
+special circumstances.
+
+  For example, on rare occasions, there may be a special need to
+encourage the widest possible use of a certain library, so that it
+becomes a de-facto standard.  To achieve this, non-free programs must
+be allowed to use the library.  A more frequent case is that a free
+library does the same job as widely used non-free libraries.  In this
+case, there is little to gain by limiting the free library to free
+software only, so we use the Lesser General Public License.
+
+  In other cases, permission to use a particular library in non-free
+programs enables a greater number of people to use a large body of
+free software.  For example, permission to use the GNU C Library in
+non-free programs enables many more people to use the whole GNU
+operating system, as well as its variant, the GNU/Linux operating
+system.
+
+  Although the Lesser General Public License is Less protective of the
+users' freedom, it does ensure that the user of a program that is
+linked with the Library has the freedom and the wherewithal to run
+that program using a modified version of the Library.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.  Pay close attention to the difference between a
+"work based on the library" and a "work that uses the library".  The
+former contains code derived from the library, whereas the latter must
+be combined with the library in order to run.
+
+                  GNU LESSER GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License Agreement applies to any software library or other
+program which contains a notice placed by the copyright holder or
+other authorized party saying it may be distributed under the terms of
+this Lesser General Public License (also called "this License").
+Each licensee is addressed as "you".
+
+  A "library" means a collection of software functions and/or data
+prepared so as to be conveniently linked with application programs
+(which use some of those functions and data) to form executables.
+
+  The "Library", below, refers to any such software library or work
+which has been distributed under these terms.  A "work based on the
+Library" means either the Library or any derivative work under
+copyright law: that is to say, a work containing the Library or a
+portion of it, either verbatim or with modifications and/or translated
+straightforwardly into another language.  (Hereinafter, translation is
+included without limitation in the term "modification".)
+
+  "Source code" for a work means the preferred form of the work for
+making modifications to it.  For a library, complete source code means
+all the source code for all modules it contains, plus any associated
+interface definition files, plus the scripts used to control
+compilation and installation of the library.
+
+  Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running a program using the Library is not restricted, and output from
+such a program is covered only if its contents constitute a work based
+on the Library (independent of the use of the Library in a tool for
+writing it).  Whether that is true depends on what the Library does
+and what the program that uses the Library does.
+
+  1. You may copy and distribute verbatim copies of the Library's
+complete source code as you receive it, in any medium, provided that
+you conspicuously and appropriately publish on each copy an
+appropriate copyright notice and disclaimer of warranty; keep intact
+all the notices that refer to this License and to the absence of any
+warranty; and distribute a copy of this License along with the
+Library.
+
+  You may charge a fee for the physical act of transferring a copy,
+and you may at your option offer warranty protection in exchange for a
+fee.
+
+  2. You may modify your copy or copies of the Library or any portion
+of it, thus forming a work based on the Library, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) The modified work must itself be a software library.
+
+    b) You must cause the files modified to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    c) You must cause the whole of the work to be licensed at no
+    charge to all third parties under the terms of this License.
+
+    d) If a facility in the modified Library refers to a function or a
+    table of data to be supplied by an application program that uses
+    the facility, other than as an argument passed when the facility
+    is invoked, then you must make a good faith effort to ensure that,
+    in the event an application does not supply such function or
+    table, the facility still operates, and performs whatever part of
+    its purpose remains meaningful.
+
+    (For example, a function in a library to compute square roots has
+    a purpose that is entirely well-defined independent of the
+    application.  Therefore, Subsection 2d requires that any
+    application-supplied function or table used by this function must
+    be optional: if the application does not supply it, the square
+    root function must still compute square roots.)
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Library,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Library, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote
+it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Library.
+
+In addition, mere aggregation of another work not based on the Library
+with the Library (or with a work based on the Library) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may opt to apply the terms of the ordinary GNU General Public
+License instead of this License to a given copy of the Library.  To do
+this, you must alter all the notices that refer to this License, so
+that they refer to the ordinary GNU General Public License, version 2,
+instead of to this License.  (If a newer version than version 2 of the
+ordinary GNU General Public License has appeared, then you can specify
+that version instead if you wish.)  Do not make any other change in
+these notices.
+
+  Once this change is made in a given copy, it is irreversible for
+that copy, so the ordinary GNU General Public License applies to all
+subsequent copies and derivative works made from that copy.
+
+  This option is useful when you wish to copy part of the code of
+the Library into a program that is not a library.
+
+  4. You may copy and distribute the Library (or a portion or
+derivative of it, under Section 2) in object code or executable form
+under the terms of Sections 1 and 2 above provided that you accompany
+it with the complete corresponding machine-readable source code, which
+must be distributed under the terms of Sections 1 and 2 above on a
+medium customarily used for software interchange.
+
+  If distribution of object code is made by offering access to copy
+from a designated place, then offering equivalent access to copy the
+source code from the same place satisfies the requirement to
+distribute the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+  5. A program that contains no derivative of any portion of the
+Library, but is designed to work with the Library by being compiled or
+linked with it, is called a "work that uses the Library".  Such a
+work, in isolation, is not a derivative work of the Library, and
+therefore falls outside the scope of this License.
+
+  However, linking a "work that uses the Library" with the Library
+creates an executable that is a derivative of the Library (because it
+contains portions of the Library), rather than a "work that uses the
+library".  The executable is therefore covered by this License.
+Section 6 states terms for distribution of such executables.
+
+  When a "work that uses the Library" uses material from a header file
+that is part of the Library, the object code for the work may be a
+derivative work of the Library even though the source code is not.
+Whether this is true is especially significant if the work can be
+linked without the Library, or if the work is itself a library.  The
+threshold for this to be true is not precisely defined by law.
+
+  If such an object file uses only numerical parameters, data
+structure layouts and accessors, and small macros and small inline
+functions (ten lines or less in length), then the use of the object
+file is unrestricted, regardless of whether it is legally a derivative
+work.  (Executables containing this object code plus portions of the
+Library will still fall under Section 6.)
+
+  Otherwise, if the work is a derivative of the Library, you may
+distribute the object code for the work under the terms of Section 6.
+Any executables containing that work also fall under Section 6,
+whether or not they are linked directly with the Library itself.
+
+  6. As an exception to the Sections above, you may also combine or
+link a "work that uses the Library" with the Library to produce a
+work containing portions of the Library, and distribute that work
+under terms of your choice, provided that the terms permit
+modification of the work for the customer's own use and reverse
+engineering for debugging such modifications.
+
+  You must give prominent notice with each copy of the work that the
+Library is used in it and that the Library and its use are covered by
+this License.  You must supply a copy of this License.  If the work
+during execution displays copyright notices, you must include the
+copyright notice for the Library among them, as well as a reference
+directing the user to the copy of this License.  Also, you must do one
+of these things:
+
+    a) Accompany the work with the complete corresponding
+    machine-readable source code for the Library including whatever
+    changes were used in the work (which must be distributed under
+    Sections 1 and 2 above); and, if the work is an executable linked
+    with the Library, with the complete machine-readable "work that
+    uses the Library", as object code and/or source code, so that the
+    user can modify the Library and then relink to produce a modified
+    executable containing the modified Library.  (It is understood
+    that the user who changes the contents of definitions files in the
+    Library will not necessarily be able to recompile the application
+    to use the modified definitions.)
+
+    b) Use a suitable shared library mechanism for linking with the
+    Library.  A suitable mechanism is one that (1) uses at run time a
+    copy of the library already present on the user's computer system,
+    rather than copying library functions into the executable, and (2)
+    will operate properly with a modified version of the library, if
+    the user installs one, as long as the modified version is
+    interface-compatible with the version that the work was made with.
+
+    c) Accompany the work with a written offer, valid for at least
+    three years, to give the same user the materials specified in
+    Subsection 6a, above, for a charge no more than the cost of
+    performing this distribution.
+
+    d) If distribution of the work is made by offering access to copy
+    from a designated place, offer equivalent access to copy the above
+    specified materials from the same place.
+
+    e) Verify that the user has already received a copy of these
+    materials or that you have already sent this user a copy.
+
+  For an executable, the required form of the "work that uses the
+Library" must include any data and utility programs needed for
+reproducing the executable from it.  However, as a special exception,
+the materials to be distributed need not include anything that is
+normally distributed (in either source or binary form) with the major
+components (compiler, kernel, and so on) of the operating system on
+which the executable runs, unless that component itself accompanies
+the executable.
+
+  It may happen that this requirement contradicts the license
+restrictions of other proprietary libraries that do not normally
+accompany the operating system.  Such a contradiction means you cannot
+use both them and the Library together in an executable that you
+distribute.
+
+  7. You may place library facilities that are a work based on the
+Library side-by-side in a single library together with other library
+facilities not covered by this License, and distribute such a combined
+library, provided that the separate distribution of the work based on
+the Library and of the other library facilities is otherwise
+permitted, and provided that you do these two things:
+
+    a) Accompany the combined library with a copy of the same work
+    based on the Library, uncombined with any other library
+    facilities.  This must be distributed under the terms of the
+    Sections above.
+
+    b) Give prominent notice with the combined library of the fact
+    that part of it is a work based on the Library, and explaining
+    where to find the accompanying uncombined form of the same work.
+
+  8. You may not copy, modify, sublicense, link with, or distribute
+the Library except as expressly provided under this License.  Any
+attempt otherwise to copy, modify, sublicense, link with, or
+distribute the Library is void, and will automatically terminate your
+rights under this License.  However, parties who have received copies,
+or rights, from you under this License will not have their licenses
+terminated so long as such parties remain in full compliance.
+
+  9. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Library or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Library (or any work based on the
+Library), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Library or works based on it.
+
+  10. Each time you redistribute the Library (or any work based on the
+Library), the recipient automatically receives a license from the
+original licensor to copy, distribute, link with or modify the Library
+subject to these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties with
+this License.
+
+  11. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Library at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Library by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Library.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply, and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+  12. If the distribution and/or use of the Library is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Library under this License
+may add an explicit geographical distribution limitation excluding those
+countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  13. The Free Software Foundation may publish revised and/or new
+versions of the Lesser General Public License from time to time.
+Such new versions will be similar in spirit to the present version,
+but may differ in detail to address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Library
+specifies a version number of this License which applies to it and
+"any later version", you have the option of following the terms and
+conditions either of that version or of any later version published by
+the Free Software Foundation.  If the Library does not specify a
+license version number, you may choose any version ever published by
+the Free Software Foundation.
+
+  14. If you wish to incorporate parts of the Library into other free
+programs whose distribution conditions are incompatible with these,
+write to the author to ask for permission.  For software which is
+copyrighted by the Free Software Foundation, write to the Free
+Software Foundation; we sometimes make exceptions for this.  Our
+decision will be guided by the two goals of preserving the free status
+of all derivatives of our free software and of promoting the sharing
+and reuse of software generally.
+
+                            NO WARRANTY
+
+  15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO
+WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW.
+EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR
+OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY
+KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE
+LIBRARY IS WITH YOU.  SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME
+THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN
+WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY
+AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU
+FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR
+CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE
+LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING
+RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A
+FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF
+SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
+DAMAGES.
+
+                     END OF TERMS AND CONDITIONS
+
+           How to Apply These Terms to Your New Libraries
+
+  If you develop a new library, and you want it to be of the greatest
+possible use to the public, we recommend making it free software that
+everyone can redistribute and change.  You can do so by permitting
+redistribution under these terms (or, alternatively, under the terms
+of the ordinary General Public License).
+
+  To apply these terms, attach the following notices to the library.
+It is safest to attach them to the start of each source file to most
+effectively convey the exclusion of warranty; and each file should
+have at least the "copyright" line and a pointer to where the full
+notice is found.
+
+
+    <one line to give the library's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation; either
+    version 2.1 of the License, or (at your option) any later version.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+Also add information on how to contact you by electronic and paper mail.
+
+You should also get your employer (if you work as a programmer) or
+your school, if any, to sign a "copyright disclaimer" for the library,
+if necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the
+  library `Frob' (a library for tweaking knobs) written by James
+  Random Hacker.
+
+  <signature of Ty Coon>, 1 April 1990
+  Ty Coon, President of Vice
+
+That's all there is to it!
+
+

pylibs/logilab/astng/README

@@ -0,0 +1,54 @@
+ASTNG
+=====
+
+What's this ?
+-------------
+
+The aim of this module is to provide a common base representation of
+python source code for projects such as pychecker, pyreverse,
+pylint... Well, actually the development of this library is essentially
+governed by pylint's needs.
+
+It provides a compatible representation which comes from the `_ast` module.
+It rebuilds the tree generated by the builtin _ast module by recursively
+walking down the AST and building an extended ast (let's call it astng ;). The
+new node classes have additional methods and attributes for different usages.
+They include some support for static inference and local name scopes.
+Furthermore, astng builds partial trees by inspecting living objects.
+
+Main modules are:
+
+* `bases`, `node_classses` and `scoped_nodes` contain the classes for the
+  different type of nodes of the tree.
+
+* the `manager` contains a high level object to get astng trees from
+  source files and living objects. It maintains a cache of previously
+  constructed tree for quick access
+
+
+Installation
+------------
+
+Extract the tarball, jump into the created directory and run ::
+
+	python setup.py install
+
+For installation options, see ::
+
+	python setup.py install --help
+
+
+If you have any questions, please mail the
+python-project@lists.logilab.org mailing list for support. See
+http://lists.logilab.org/mailman/listinfo/python-projects for
+subscription information and archives.
+
+Test
+----
+
+Tests are in the 'test' subdirectory. To launch the whole tests suite
+at once, you may use the 'pytest' utility from logilab-common (simply
+type 'pytest' from within this directory) or if you're running python
+>= 2.7, using discover, for instance::
+
+  python -m unittest discover -p "unittest*.py"

pylibs/logilab/astng/README.Python3

@@ -0,0 +1,26 @@
+Python3
+=======
+
+Approach
+--------
+
+We maintain a Python 2 base and use 2to3 to generate Python 3 code.
+
+2to3 is integrated into the distutils installation process and will be run as a
+build step when invoked by the python3 interpreter::
+
+  python3 setup.py install --no-compile
+
+
+Debian
+------
+
+For the Debian packaging, you can use the debian.py3k/ content against
+the debian/ folder::
+
+  cp debian.py3k/* debian/
+
+
+Resources
+---------
+http://wiki.python.org/moin/PortingPythonToPy3k

pylibs/logilab/astng/__init__.py

@@ -0,0 +1,85 @@
+# copyright 2003-2012 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+# copyright 2003-2010 Sylvain Thenault, all rights reserved.
+# contact mailto:thenault@gmail.com
+#
+# This file is part of logilab-astng.
+#
+# logilab-astng is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Lesser General Public License as published by the
+# Free Software Foundation, either version 2.1 of the License, or (at your
+# option) any later version.
+#
+# logilab-astng is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License
+# for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with logilab-astng. If not, see <http://www.gnu.org/licenses/>.
+"""Python Abstract Syntax Tree New Generation
+
+The aim of this module is to provide a common base representation of
+python source code for projects such as pychecker, pyreverse,
+pylint... Well, actually the development of this library is essentially
+governed by pylint's needs.
+
+It extends class defined in the python's _ast module with some
+additional methods and attributes. Instance attributes are added by a
+builder object, which can either generate extended ast (let's call
+them astng ;) by visiting an existent ast tree or by inspecting living
+object. Methods are added by monkey patching ast classes.
+
+Main modules are:
+
+* nodes and scoped_nodes for more information about methods and
+  attributes added to different node classes
+
+* the manager contains a high level object to get astng trees from
+  source files and living objects. It maintains a cache of previously
+  constructed tree for quick access
+
+* builder contains the class responsible to build astng trees
+"""
+__doctype__ = "restructuredtext en"
+
+import sys
+if sys.version_info >= (3, 0):
+    BUILTINS_MODULE = 'builtins'
+else:
+    BUILTINS_MODULE = '__builtin__'
+
+# WARNING: internal imports order matters !
+
+# make all exception classes accessible from astng package
+from logilab.astng.exceptions import *
+
+# make all node classes accessible from astng package
+from logilab.astng.nodes import *
+
+# trigger extra monkey-patching
+from logilab.astng import inference
+
+# more stuff available
+from logilab.astng import raw_building
+from logilab.astng.bases import YES, Instance, BoundMethod, UnboundMethod
+from logilab.astng.node_classes import are_exclusive, unpack_infer
+from logilab.astng.scoped_nodes import builtin_lookup
+
+# make a manager instance (borg) as well as Project and Package classes
+# accessible from astng package
+from logilab.astng.manager import ASTNGManager, Project
+MANAGER = ASTNGManager()
+del ASTNGManager
+
+# load brain plugins
+from os import listdir
+from os.path import join, dirname
+BRAIN_MODULES_DIR = join(dirname(__file__), 'brain')
+if BRAIN_MODULES_DIR not in sys.path:
+    # add it to the end of the list so user path take precedence
+    sys.path.append(BRAIN_MODULES_DIR)
+# load modules in this directory
+for module in listdir(BRAIN_MODULES_DIR):
+    if module.endswith('.py'):
+        __import__(module[:-3])

pylibs/logilab/astng/__pkginfo__.py

@@ -0,0 +1,45 @@
+# copyright 2003-2012 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+# copyright 2003-2010 Sylvain Thenault, all rights reserved.
+# contact mailto:thenault@gmail.com
+#
+# This file is part of logilab-astng.
+#
+# logilab-astng is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Lesser General Public License as published by the
+# Free Software Foundation, either version 2.1 of the License, or (at your
+# option) any later version.
+#
+# logilab-astng is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License
+# for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with logilab-astng. If not, see <http://www.gnu.org/licenses/>.
+"""logilab.astng packaging information"""
+
+distname = 'logilab-astng'
+
+modname = 'astng'
+subpackage_of = 'logilab'
+
+numversion = (0, 24, 0)
+version = '.'.join([str(num) for num in numversion])
+
+install_requires = ['logilab-common >= 0.53.0']
+
+license = 'LGPL'
+
+author = 'Logilab'
+author_email = 'python-projects@lists.logilab.org'
+mailinglist = "mailto://%s" % author_email
+web = "http://www.logilab.org/project/%s" % distname
+ftp = "ftp://ftp.logilab.org/pub/%s" % modname
+
+description = "rebuild a new abstract syntax tree from Python's ast"
+
+from os.path import join
+include_dirs = ['brain',
+                join('test', 'regrtest_data'),
+                join('test', 'data'), join('test', 'data2')]

pylibs/logilab/astng/as_string.py

@@ -0,0 +1,427 @@
+# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+# copyright 2003-2010 Sylvain Thenault, all rights reserved.
+# contact mailto:thenault@gmail.com
+#
+# This file is part of logilab-astng.
+#
+# logilab-astng is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Lesser General Public License as published by the
+# Free Software Foundation, either version 2.1 of the License, or (at your
+# option) any later version.
+#
+# logilab-astng is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License
+# for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with logilab-astng. If not, see <http://www.gnu.org/licenses/>.
+"""This module renders ASTNG nodes to string representation.
+
+It will probably not work on bare _ast trees.
+"""
+import sys
+
+
+INDENT = '    ' # 4 spaces ; keep indentation variable
+
+
+def _import_string(names):
+    """return a list of (name, asname) formatted as a string"""
+    _names = []
+    for name, asname in names:
+        if asname is not None:
+            _names.append('%s as %s' % (name, asname))
+        else:
+            _names.append(name)
+    return  ', '.join(_names)
+
+
+class AsStringVisitor(object):
+    """Visitor to render an ASTNG node as string """
+
+    def __call__(self, node):
+        """Makes this visitor behave as a simple function"""
+        return node.accept(self)
+
+    def _stmt_list(self, stmts):
+        """return a list of nodes to string"""
+        stmts = '\n'.join([nstr for nstr in [n.accept(self) for n in stmts] if nstr])
+        return INDENT + stmts.replace('\n', '\n'+INDENT)
+
+
+    ## visit_<node> methods ###########################################
+
+    def visit_arguments(self, node):
+        """return an astng.Function node as string"""
+        return node.format_args()
+
+    def visit_assattr(self, node):
+        """return an astng.AssAttr node as string"""
+        return self.visit_getattr(node)
+
+    def visit_assert(self, node):
+        """return an astng.Assert node as string"""
+        if node.fail:
+            return 'assert %s, %s' % (node.test.accept(self),
+                                        node.fail.accept(self))
+        return 'assert %s' % node.test.accept(self)
+
+    def visit_assname(self, node):
+        """return an astng.AssName node as string"""
+        return node.name
+
+    def visit_assign(self, node):
+        """return an astng.Assign node as string"""
+        lhs = ' = '.join([n.accept(self) for n in node.targets])
+        return '%s = %s' % (lhs, node.value.accept(self))
+
+    def visit_augassign(self, node):
+        """return an astng.AugAssign node as string"""
+        return '%s %s %s' % (node.target.accept(self), node.op, node.value.accept(self))
+
+    def visit_backquote(self, node):
+        """return an astng.Backquote node as string"""
+        return '`%s`' % node.value.accept(self)
+
+    def visit_binop(self, node):
+        """return an astng.BinOp node as string"""
+        return '(%s) %s (%s)' % (node.left.accept(self), node.op, node.right.accept(self))
+
+    def visit_boolop(self, node):
+        """return an astng.BoolOp node as string"""
+        return (' %s ' % node.op).join(['(%s)' % n.accept(self)
+                                            for n in node.values])
+
+    def visit_break(self, node):
+        """return an astng.Break node as string"""
+        return 'break'
+
+    def visit_callfunc(self, node):
+        """return an astng.CallFunc node as string"""
+        expr_str = node.func.accept(self)
+        args = [arg.accept(self) for arg in node.args]
+        if node.starargs:
+            args.append( '*' + node.starargs.accept(self))
+        if node.kwargs:
+            args.append( '**' + node.kwargs.accept(self))
+        return '%s(%s)' % (expr_str, ', '.join(args))
+
+    def visit_class(self, node):
+        """return an astng.Class node as string"""
+        decorate = node.decorators and node.decorators.accept(self)  or ''
+        bases =  ', '.join([n.accept(self) for n in node.bases])
+        bases = bases and '(%s)' % bases or ''
+        docs = node.doc and '\n%s"""%s"""' % (INDENT, node.doc) or ''
+        return '\n\n%sclass %s%s:%s\n%s\n' % (decorate, node.name, bases, docs,
+                                            self._stmt_list( node.body))
+
+    def visit_compare(self, node):
+        """return an astng.Compare node as string"""
+        rhs_str = ' '.join(['%s %s' % (op, expr.accept(self))
+                            for op, expr in node.ops])
+        return '%s %s' % (node.left.accept(self), rhs_str)
+
+    def visit_comprehension(self, node):
+        """return an astng.Comprehension node as string"""
+        ifs = ''.join([ ' if %s' % n.accept(self) for n in node.ifs])
+        return 'for %s in %s%s' % (node.target.accept(self),
+                                    node.iter.accept(self), ifs )
+
+    def visit_const(self, node):
+        """return an astng.Const node as string"""
+        return repr(node.value)
+
+    def visit_continue(self, node):
+        """return an astng.Continue node as string"""
+        return 'continue'
+
+    def visit_delete(self, node): # XXX check if correct
+        """return an astng.Delete node as string"""
+        return 'del %s' % ', '.join([child.accept(self)
+                                for child in node.targets])
+
+    def visit_delattr(self, node):
+        """return an astng.DelAttr node as string"""
+        return self.visit_getattr(node)
+
+    def visit_delname(self, node):
+        """return an astng.DelName node as string"""
+        return node.name
+
+    def visit_decorators(self, node):
+        """return an astng.Decorators node as string"""
+        return '@%s\n' % '\n@'.join([item.accept(self) for item in node.nodes])
+
+    def visit_dict(self, node):
+        """return an astng.Dict node as string"""
+        return '{%s}' % ', '.join(['%s: %s' % (key.accept(self),
+                            value.accept(self)) for key, value in node.items])
+
+    def visit_dictcomp(self, node):
+        """return an astng.DictComp node as string"""
+        return '{%s: %s %s}' % (node.key.accept(self), node.value.accept(self),
+                ' '.join([n.accept(self) for n in node.generators]))
+
+    def visit_discard(self, node):
+        """return an astng.Discard node as string"""
+        return node.value.accept(self)
+
+    def visit_emptynode(self, node):
+        """dummy method for visiting an Empty node"""
+        return ''
+
+    def visit_excepthandler(self, node):
+        if node.type:
+            if node.name:
+                excs = 'except %s, %s' % (node.type.accept(self),
+                                        node.name.accept(self))
+            else:
+                excs = 'except %s' % node.type.accept(self)
+        else:
+            excs = 'except'
+        return '%s:\n%s' % (excs, self._stmt_list(node.body))
+
+    def visit_ellipsis(self, node):
+        """return an astng.Ellipsis node as string"""
+        return '...'
+
+    def visit_empty(self, node):
+        """return an Empty node as string"""
+        return ''
+
+    def visit_exec(self, node):
+        """return an astng.Exec node as string"""
+        if node.locals:
+            return 'exec %s in %s, %s' % (node.expr.accept(self),
+                                          node.locals.accept(self),
+                                          node.globals.accept(self))
+        if node.globals:
+            return 'exec %s in %s' % (node.expr.accept(self),
+                                      node.globals.accept(self))
+        return 'exec %s' % node.expr.accept(self)
+
+    def visit_extslice(self, node):
+        """return an astng.ExtSlice node as string"""
+        return ','.join( [dim.accept(self) for dim in node.dims] )
+
+    def visit_for(self, node):
+        """return an astng.For node as string"""
+        fors = 'for %s in %s:\n%s' % (node.target.accept(self),
+                                    node.iter.accept(self),
+                                    self._stmt_list( node.body))
+        if node.orelse:
+            fors = '%s\nelse:\n%s' % (fors, self._stmt_list(node.orelse))
+        return fors
+
+    def visit_from(self, node):
+        """return an astng.From node as string"""
+        return 'from %s import %s' % ('.' * (node.level or 0) + node.modname,
+                                      _import_string(node.names))
+
+    def visit_function(self, node):
+        """return an astng.Function node as string"""
+        decorate = node.decorators and node.decorators.accept(self)  or ''
+        docs = node.doc and '\n%s"""%s"""' % (INDENT, node.doc) or ''
+        return '\n%sdef %s(%s):%s\n%s' % (decorate, node.name, node.args.accept(self),
+                                        docs, self._stmt_list(node.body))
+
+    def visit_genexpr(self, node):
+        """return an astng.GenExpr node as string"""
+        return '(%s %s)' % (node.elt.accept(self), ' '.join([n.accept(self)
+                                                    for n in node.generators]))
+
+    def visit_getattr(self, node):
+        """return an astng.Getattr node as string"""
+        return '%s.%s' % (node.expr.accept(self), node.attrname)
+
+    def visit_global(self, node):
+        """return an astng.Global node as string"""
+        return 'global %s' % ', '.join(node.names)
+
+    def visit_if(self, node):
+        """return an astng.If node as string"""
+        ifs = ['if %s:\n%s' % (node.test.accept(self), self._stmt_list(node.body))]
+        if node.orelse:# XXX use elif ???
+            ifs.append('else:\n%s' % self._stmt_list(node.orelse))
+        return '\n'.join(ifs)
+
+    def visit_ifexp(self, node):
+        """return an astng.IfExp node as string"""
+        return '%s if %s else %s' % (node.body.accept(self),
+                node.test.accept(self), node.orelse.accept(self))
+
+    def visit_import(self, node):
+        """return an astng.Import node as string"""
+        return 'import %s' % _import_string(node.names)
+
+    def visit_keyword(self, node):
+        """return an astng.Keyword node as string"""
+        return '%s=%s' % (node.arg, node.value.accept(self))
+
+    def visit_lambda(self, node):
+        """return an astng.Lambda node as string"""
+        return 'lambda %s: %s' % (node.args.accept(self), node.body.accept(self))
+
+    def visit_list(self, node):
+        """return an astng.List node as string"""
+        return '[%s]' % ', '.join([child.accept(self) for child in node.elts])
+
+    def visit_listcomp(self, node):
+        """return an astng.ListComp node as string"""
+        return '[%s %s]' % (node.elt.accept(self), ' '.join([n.accept(self)
+                                                for n in node.generators]))
+
+    def visit_module(self, node):
+        """return an astng.Module node as string"""
+        docs = node.doc and '"""%s"""\n\n' % node.doc or ''
+        return docs + '\n'.join([n.accept(self) for n in node.body]) + '\n\n'
+
+    def visit_name(self, node):
+        """return an astng.Name node as string"""
+        return node.name
+
+    def visit_pass(self, node):
+        """return an astng.Pass node as string"""
+        return 'pass'
+
+    def visit_print(self, node):
+        """return an astng.Print node as string"""
+        nodes = ', '.join([n.accept(self) for n in node.values])
+        if not node.nl:
+            nodes = '%s,' % nodes
+        if node.dest:
+            return 'print >> %s, %s' % (node.dest.accept(self), nodes)
+        return 'print %s' % nodes
+
+    def visit_raise(self, node):
+        """return an astng.Raise node as string"""
+        if node.exc:
+            if node.inst:
+                if node.tback:
+                    return 'raise %s, %s, %s' % (node.exc.accept(self),
+                                                node.inst.accept(self),
+                                                node.tback.accept(self))
+                return 'raise %s, %s' % (node.exc.accept(self),
+                                        node.inst.accept(self))
+            return 'raise %s' % node.exc.accept(self)
+        return 'raise'
+
+    def visit_return(self, node):
+        """return an astng.Return node as string"""
+        if node.value:
+            return 'return %s' % node.value.accept(self)
+        else:
+            return 'return'
+
+    def visit_index(self, node):
+        """return a astng.Index node as string"""
+        return node.value.accept(self)
+
+    def visit_set(self, node):
+        """return an astng.Set node as string"""
+        return '{%s}' % ', '.join([child.accept(self) for child in node.elts])
+
+    def visit_setcomp(self, node):
+        """return an astng.SetComp node as string"""
+        return '{%s %s}' % (node.elt.accept(self), ' '.join([n.accept(self)
+                                                for n in node.generators]))
+
+    def visit_slice(self, node):
+        """return a astng.Slice node as string"""
+        lower = node.lower and node.lower.accept(self) or ''
+        upper = node.upper and node.upper.accept(self) or ''
+        step = node.step and node.step.accept(self) or ''
+        if step:
+            return '%s:%s:%s' % (lower, upper, step)
+        return  '%s:%s' % (lower, upper)
+
+    def visit_subscript(self, node):
+        """return an astng.Subscript node as string"""
+        return '%s[%s]' % (node.value.accept(self), node.slice.accept(self))
+
+    def visit_tryexcept(self, node):
+        """return an astng.TryExcept node as string"""
+        trys = ['try:\n%s' % self._stmt_list( node.body)]
+        for handler in node.handlers:
+            trys.append(handler.accept(self))
+        if node.orelse:
+            trys.append('else:\n%s' % self._stmt_list(node.orelse))
+        return '\n'.join(trys)
+
+    def visit_tryfinally(self, node):
+        """return an astng.TryFinally node as string"""
+        return 'try:\n%s\nfinally:\n%s' % (self._stmt_list( node.body),
+                                        self._stmt_list(node.finalbody))
+
+    def visit_tuple(self, node):
+        """return an astng.Tuple node as string"""
+        return '(%s)' % ', '.join([child.accept(self) for child in node.elts])
+
+    def visit_unaryop(self, node):
+        """return an astng.UnaryOp node as string"""
+        if node.op == 'not':
+            operator = 'not '
+        else:
+            operator = node.op
+        return '%s%s' % (operator, node.operand.accept(self))
+
+    def visit_while(self, node):
+        """return an astng.While node as string"""
+        whiles = 'while %s:\n%s' % (node.test.accept(self),
+                                    self._stmt_list(node.body))
+        if node.orelse:
+            whiles = '%s\nelse:\n%s' % (whiles, self._stmt_list(node.orelse))
+        return whiles
+
+    def visit_with(self, node): # 'with' without 'as' is possible
+        """return an astng.With node as string"""
+        as_var = node.vars and " as (%s)" % (node.vars.accept(self)) or ""
+        withs = 'with (%s)%s:\n%s' % (node.expr.accept(self), as_var,
+                                        self._stmt_list( node.body))
+        return withs
+
+    def visit_yield(self, node):
+        """yield an ast.Yield node as string"""
+        yi_val = node.value and (" " + node.value.accept(self)) or ""
+        return 'yield' + yi_val
+
+
+class AsStringVisitor3k(AsStringVisitor):
+    """AsStringVisitor3k overwrites some AsStringVisitor methods"""
+
+    def visit_excepthandler(self, node):
+        if node.type:
+            if node.name:
+                excs = 'except %s as %s' % (node.type.accept(self),
+                                        node.name.accept(self))
+            else:
+                excs = 'except %s' % node.type.accept(self)
+        else:
+            excs = 'except'
+        return '%s:\n%s' % (excs, self._stmt_list(node.body))
+
+    def visit_nonlocal(self, node):
+        """return an astng.Nonlocal node as string"""
+        return 'nonlocal %s' % ', '.join(node.names)
+
+    def visit_raise(self, node):
+        """return an astng.Raise node as string"""
+        if node.exc:
+            if node.cause:
+                return 'raise %s from %s' % (node.exc.accept(self),
+                                             node.cause.accept(self))
+            return 'raise %s' % node.exc.accept(self)
+        return 'raise'
+
+    def visit_starred(self, node):
+        """return Starred node as string"""
+        return "*" + node.value.accept(self)
+
+if sys.version_info >= (3, 0):
+    AsStringVisitor = AsStringVisitor3k
+
+# this visitor is stateless, thus it can be reused
+as_string = AsStringVisitor()
+

pylibs/logilab/astng/bases.py

@@ -0,0 +1,629 @@
+# -*- coding: utf-8 -*-
+# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+# copyright 2003-2010 Sylvain Thenault, all rights reserved.
+# contact mailto:thenault@gmail.com
+#
+# This file is part of logilab-astng.
+#
+# logilab-astng is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Lesser General Public License as published by the
+# Free Software Foundation, either version 2.1 of the License, or (at your
+# option) any later version.
+#
+# logilab-astng is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License
+# for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with logilab-astng. If not, see <http://www.gnu.org/licenses/>.
+"""This module contains base classes and functions for the nodes and some
+inference utils.
+"""
+
+__docformat__ = "restructuredtext en"
+
+from contextlib import contextmanager
+
+from logilab.common.compat import builtins
+
+from logilab.astng import BUILTINS_MODULE
+from logilab.astng.exceptions import InferenceError, ASTNGError, \
+                                       NotFoundError, UnresolvableName
+from logilab.astng.as_string import as_string
+
+BUILTINS_NAME = builtins.__name__
+
+class Proxy(object):
+    """a simple proxy object"""
+    _proxied = None
+
+    def __init__(self, proxied=None):
+        if proxied is not None:
+            self._proxied = proxied
+
+    def __getattr__(self, name):
+        if name == '_proxied':
+            return getattr(self.__class__, '_proxied')
+        if name in self.__dict__:
+            return self.__dict__[name]
+        return getattr(self._proxied, name)
+
+    def infer(self, context=None):
+        yield self
+
+
+# Inference ##################################################################
+
+class InferenceContext(object):
+    __slots__ = ('path', 'lookupname', 'callcontext', 'boundnode')
+
+    def __init__(self, path=None):
+        if path is None:
+            self.path = set()
+        else:
+            self.path = path
+        self.lookupname = None
+        self.callcontext = None
+        self.boundnode = None
+
+    def push(self, node):
+        name = self.lookupname
+        if (node, name) in self.path:
+            raise StopIteration()
+        self.path.add( (node, name) )
+
+    def clone(self):
+        # XXX copy lookupname/callcontext ?
+        clone = InferenceContext(self.path)
+        clone.callcontext = self.callcontext
+        clone.boundnode = self.boundnode
+        return clone
+
+    @contextmanager
+    def restore_path(self):
+        path = set(self.path)
+        yield
+        self.path = path
+
+def copy_context(context):
+    if context is not None:
+        return context.clone()
+    else:
+        return InferenceContext()
+
+
+def _infer_stmts(stmts, context, frame=None):
+    """return an iterator on statements inferred by each statement in <stmts>
+    """
+    stmt = None
+    infered = False
+    if context is not None:
+        name = context.lookupname
+        context = context.clone()
+    else:
+        name = None
+        context = InferenceContext()
+    for stmt in stmts:
+        if stmt is YES:
+            yield stmt
+            infered = True
+            continue
+        context.lookupname = stmt._infer_name(frame, name)
+        try:
+            for infered in stmt.infer(context):
+                yield infered
+                infered = True
+        except UnresolvableName:
+            continue
+        except InferenceError:
+            yield YES
+            infered = True
+    if not infered:
+        raise InferenceError(str(stmt))
+
+
+# special inference objects (e.g. may be returned as nodes by .infer()) #######
+
+class _Yes(object):
+    """a yes object"""
+    def __repr__(self):
+        return 'YES'
+    def __getattribute__(self, name):
+        if name.startswith('__') and name.endswith('__'):
+            # to avoid inspection pb
+            return super(_Yes, self).__getattribute__(name)
+        return self
+    def __call__(self, *args, **kwargs):
+        return self
+
+
+YES = _Yes()
+
+
+class Instance(Proxy):
+    """a special node representing a class instance"""
+    def getattr(self, name, context=None, lookupclass=True):
+        try:
+            values = self._proxied.instance_attr(name, context)
+        except NotFoundError:
+            if name == '__class__':
+                return [self._proxied]
+            if lookupclass:
+                # class attributes not available through the instance
+                # unless they are explicitly defined
+                if name in ('__name__', '__bases__', '__mro__', '__subclasses__'):
+                    return self._proxied.local_attr(name)
+                return self._proxied.getattr(name, context)
+            raise NotFoundError(name)
+        # since we've no context information, return matching class members as
+        # well
+        if lookupclass:
+            try:
+                return values + self._proxied.getattr(name, context)
+            except NotFoundError:
+                pass
+        return values
+
+    def igetattr(self, name, context=None):
+        """inferred getattr"""
+        try:
+            # XXX frame should be self._proxied, or not ?
+            get_attr = self.getattr(name, context, lookupclass=False)
+            return _infer_stmts(self._wrap_attr(get_attr, context), context,
+                                frame=self)
+        except NotFoundError:
+            try:
+                # fallback to class'igetattr since it has some logic to handle
+                # descriptors
+                return self._wrap_attr(self._proxied.igetattr(name, context),
+                                       context)
+            except NotFoundError:
+                raise InferenceError(name)
+
+    def _wrap_attr(self, attrs, context=None):
+        """wrap bound methods of attrs in a InstanceMethod proxies"""
+        for attr in attrs:
+            if isinstance(attr, UnboundMethod):
+                if BUILTINS_NAME + '.property' in attr.decoratornames():
+                    for infered in attr.infer_call_result(self, context):
+                        yield infered
+                else:
+                    yield BoundMethod(attr, self)
+            else:
+                yield attr
+
+    def infer_call_result(self, caller, context=None):
+        """infer what a class instance is returning when called"""
+        infered = False
+        for node in self._proxied.igetattr('__call__', context):
+            for res in node.infer_call_result(caller, context):
+                infered = True
+                yield res
+        if not infered:
+            raise InferenceError()
+
+    def __repr__(self):
+        return '<Instance of %s.%s at 0x%s>' % (self._proxied.root().name,
+                                                self._proxied.name,
+                                                id(self))
+    def __str__(self):
+        return 'Instance of %s.%s' % (self._proxied.root().name,
+                                      self._proxied.name)
+
+    def callable(self):
+        try:
+            self._proxied.getattr('__call__')
+            return True
+        except NotFoundError:
+            return False
+
+    def pytype(self):
+        return self._proxied.qname()
+
+    def display_type(self):
+        return 'Instance of'
+
+
+class UnboundMethod(Proxy):
+    """a special node representing a method not bound to an instance"""
+    def __repr__(self):
+        frame = self._proxied.parent.frame()
+        return '<%s %s of %s at 0x%s' % (self.__class__.__name__,
+                                         self._proxied.name,
+                                         frame.qname(), id(self))
+
+    def is_bound(self):
+        return False
+
+    def getattr(self, name, context=None):
+        if name == 'im_func':
+            return [self._proxied]
+        return super(UnboundMethod, self).getattr(name, context)
+
+    def igetattr(self, name, context=None):
+        if name == 'im_func':
+            return iter((self._proxied,))
+        return super(UnboundMethod, self).igetattr(name, context)
+
+    def infer_call_result(self, caller, context):
+        # If we're unbound method __new__ of builtin object, the result is an
+        # instance of the class given as first argument.
+        if (self._proxied.name == '__new__' and
+                self._proxied.parent.frame().qname() == '%s.object' % BUILTINS_MODULE):
+            return (x is YES and x or Instance(x) for x in caller.args[0].infer())
+        return self._proxied.infer_call_result(caller, context)
+
+
+class BoundMethod(UnboundMethod):
+    """a special node representing a method bound to an instance"""
+    def __init__(self,  proxy, bound):
+        UnboundMethod.__init__(self, proxy)
+        self.bound = bound
+
+    def is_bound(self):
+        return True
+
+    def infer_call_result(self, caller, context):
+        context = context.clone()
+        context.boundnode = self.bound
+        return self._proxied.infer_call_result(caller, context)
+
+
+class Generator(Instance):
+    """a special node representing a generator"""
+    def callable(self):
+        return True
+
+    def pytype(self):
+        return '%s.generator' % BUILTINS_MODULE
+
+    def display_type(self):
+        return 'Generator'
+
+    def __repr__(self):
+        return '<Generator(%s) l.%s at 0x%s>' % (self._proxied.name, self.lineno, id(self))
+
+    def __str__(self):
+        return 'Generator(%s)' % (self._proxied.name)
+
+
+# decorators ##################################################################
+
+def path_wrapper(func):
+    """return the given infer function wrapped to handle the path"""
+    def wrapped(node, context=None, _func=func, **kwargs):
+        """wrapper function handling context"""
+        if context is None:
+            context = InferenceContext()
+        context.push(node)
+        yielded = set()
+        for res in _func(node, context, **kwargs):
+            # unproxy only true instance, not const, tuple, dict...
+            if res.__class__ is Instance:
+                ares = res._proxied
+            else:
+                ares = res
+            if not ares in yielded:
+                yield res
+                yielded.add(ares)
+    return wrapped
+
+def yes_if_nothing_infered(func):
+    def wrapper(*args, **kwargs):
+        infered = False
+        for node in func(*args, **kwargs):
+            infered = True
+            yield node
+        if not infered:
+            yield YES
+    return wrapper
+
+def raise_if_nothing_infered(func):
+    def wrapper(*args, **kwargs):
+        infered = False
+        for node in func(*args, **kwargs):
+            infered = True
+            yield node
+        if not infered:
+            raise InferenceError()
+    return wrapper
+
+
+# Node  ######################################################################
+
+class NodeNG(object):
+    """Base Class for all ASTNG node classes.
+
+    It represents a node of the new abstract syntax tree.
+    """
+    is_statement = False
+    optional_assign = False # True  for For (and for Comprehension if py <3.0)
+    is_function = False # True for Function nodes
+    # attributes below are set by the builder module or by raw factories
+    lineno = None
+    fromlineno = None
+    tolineno = None
+    col_offset = None
+    # parent node in the tree
+    parent = None
+    # attributes containing child node(s) redefined in most concrete classes:
+    _astng_fields = ()
+
+    def _repr_name(self):
+        """return self.name or self.attrname or '' for nice representation"""
+        return getattr(self, 'name', getattr(self, 'attrname', ''))
+
+    def __str__(self):
+        return '%s(%s)' % (self.__class__.__name__, self._repr_name())
+
+    def __repr__(self):
+        return '<%s(%s) l.%s [%s] at Ox%x>' % (self.__class__.__name__,
+                                           self._repr_name(),
+                                           self.fromlineno,
+                                           self.root().name,
+                                           id(self))
+
+
+    def accept(self, visitor):
+        klass = self.__class__.__name__
+        func = getattr(visitor, "visit_" + self.__class__.__name__.lower())
+        return func(self)
+
+    def get_children(self):
+        for field in self._astng_fields:
+            attr = getattr(self, field)
+            if attr is None:
+                continue
+            if isinstance(attr, (list, tuple)):
+                for elt in attr:
+                    yield elt
+            else:
+                yield attr
+
+    def last_child(self):
+        """an optimized version of list(get_children())[-1]"""
+        for field in self._astng_fields[::-1]:
+            attr = getattr(self, field)
+            if not attr: # None or empty listy / tuple
+                continue
+            if isinstance(attr, (list, tuple)):
+                return attr[-1]
+            else:
+                return attr
+        return None
+
+    def parent_of(self, node):
+        """return true if i'm a parent of the given node"""
+        parent = node.parent
+        while parent is not None:
+            if self is parent:
+                return True
+            parent = parent.parent
+        return False
+
+    def statement(self):
+        """return the first parent node marked as statement node"""
+        if self.is_statement:
+            return self
+        return self.parent.statement()
+
+    def frame(self):
+        """return the first parent frame node (i.e. Module, Function or Class)
+        """
+        return self.parent.frame()
+
+    def scope(self):
+        """return the first node defining a new scope (i.e. Module, Function,
+        Class, Lambda but also GenExpr)
+        """
+        return self.parent.scope()
+
+    def root(self):
+        """return the root node of the tree, (i.e. a Module)"""
+        if self.parent:
+            return self.parent.root()
+        return self
+
+    def child_sequence(self, child):
+        """search for the right sequence where the child lies in"""
+        for field in self._astng_fields:
+            node_or_sequence = getattr(self, field)
+            if node_or_sequence is child:
+                return [node_or_sequence]
+            # /!\ compiler.ast Nodes have an __iter__ walking over child nodes
+            if isinstance(node_or_sequence, (tuple, list)) and child in node_or_sequence:
+                return node_or_sequence
+        else:
+            msg = 'Could not found %s in %s\'s children'
+            raise ASTNGError(msg % (repr(child), repr(self)))
+
+    def locate_child(self, child):
+        """return a 2-uple (child attribute name, sequence or node)"""
+        for field in self._astng_fields:
+            node_or_sequence = getattr(self, field)
+            # /!\ compiler.ast Nodes have an __iter__ walking over child nodes
+            if child is node_or_sequence:
+                return field, child
+            if isinstance(node_or_sequence, (tuple, list)) and child in node_or_sequence:
+                return field, node_or_sequence
+        msg = 'Could not found %s in %s\'s children'
+        raise ASTNGError(msg % (repr(child), repr(self)))
+    # FIXME : should we merge child_sequence and locate_child ? locate_child
+    # is only used in are_exclusive, child_sequence one time in pylint.
+
+    def next_sibling(self):
+        """return the next sibling statement"""
+        return self.parent.next_sibling()
+
+    def previous_sibling(self):
+        """return the previous sibling statement"""
+        return self.parent.previous_sibling()
+
+    def nearest(self, nodes):
+        """return the node which is the nearest before this one in the
+        given list of nodes
+        """
+        myroot = self.root()
+        mylineno = self.fromlineno
+        nearest = None, 0
+        for node in nodes:
+            assert node.root() is myroot, \
+                   'nodes %s and %s are not from the same module' % (self, node)
+            lineno = node.fromlineno
+            if node.fromlineno > mylineno:
+                break
+            if lineno > nearest[1]:
+                nearest = node, lineno
+        # FIXME: raise an exception if nearest is None ?
+        return nearest[0]
+
+    def set_line_info(self, lastchild):
+        if self.lineno is None:
+            self.fromlineno = self._fixed_source_line()
+        else:
+            self.fromlineno = self.lineno
+        if lastchild is None:
+            self.tolineno = self.fromlineno
+        else:
+            self.tolineno = lastchild.tolineno
+        return
+        # TODO / FIXME:
+        assert self.fromlineno is not None, self
+        assert self.tolineno is not None, self
+
+    def _fixed_source_line(self):
+        """return the line number where the given node appears
+
+        we need this method since not all nodes have the lineno attribute
+        correctly set...
+        """
+        line = self.lineno
+        _node = self
+        try:
+            while line is None:
+                _node = _node.get_children().next()
+                line = _node.lineno
+        except StopIteration:
+            _node = self.parent
+            while _node and line is None:
+                line = _node.lineno
+                _node = _node.parent
+        return line
+
+    def block_range(self, lineno):
+        """handle block line numbers range for non block opening statements
+        """
+        return lineno, self.tolineno
+
+    def set_local(self, name, stmt):
+        """delegate to a scoped parent handling a locals dictionary"""
+        self.parent.set_local(name, stmt)
+
+    def nodes_of_class(self, klass, skip_klass=None):
+        """return an iterator on nodes which are instance of the given class(es)
+
+        klass may be a class object or a tuple of class objects
+        """
+        if isinstance(self, klass):
+            yield self
+        for child_node in self.get_children():
+            if skip_klass is not None and isinstance(child_node, skip_klass):
+                continue
+            for matching in child_node.nodes_of_class(klass, skip_klass):
+                yield matching
+
+    def _infer_name(self, frame, name):
+        # overridden for From, Import, Global, TryExcept and Arguments
+        return None
+
+    def infer(self, context=None):
+        """we don't know how to resolve a statement by default"""
+        # this method is overridden by most concrete classes
+        raise InferenceError(self.__class__.__name__)
+
+    def infered(self):
+        '''return list of infered values for a more simple inference usage'''
+        return list(self.infer())
+
+    def instanciate_class(self):
+        """instanciate a node if it is a Class node, else return self"""
+        return self
+
+    def has_base(self, node):
+        return False
+
+    def callable(self):
+        return False
+
+    def eq(self, value):
+        return False
+
+    def as_string(self):
+        return as_string(self)
+
+    def repr_tree(self, ids=False):
+        """print a nice astng tree representation.
+
+        :param ids: if true, we also print the ids (usefull for debugging)"""
+        result = []
+        _repr_tree(self, result, ids=ids)
+        return "\n".join(result)
+
+
+class Statement(NodeNG):
+    """Statement node adding a few attributes"""
+    is_statement = True
+
+    def next_sibling(self):
+        """return the next sibling statement"""
+        stmts = self.parent.child_sequence(self)
+        index = stmts.index(self)
+        try:
+            return stmts[index +1]
+        except IndexError:
+            pass
+
+    def previous_sibling(self):
+        """return the previous sibling statement"""
+        stmts = self.parent.child_sequence(self)
+        index = stmts.index(self)
+        if index >= 1:
+            return stmts[index -1]
+
+INDENT = "    "
+
+def _repr_tree(node, result, indent='', _done=None, ids=False):
+    """built a tree representation of a node as a list of lines"""
+    if _done is None:
+        _done = set()
+    if not hasattr(node, '_astng_fields'): # not a astng node
+        return
+    if node in _done:
+        result.append( indent + 'loop in tree: %s' % node )
+        return
+    _done.add(node)
+    node_str = str(node)
+    if ids:
+        node_str += '  . \t%x' % id(node)
+    result.append( indent + node_str )
+    indent += INDENT
+    for field in node._astng_fields:
+        value = getattr(node, field)
+        if isinstance(value, (list, tuple) ):
+            result.append(  indent + field + " = [" )
+            for child in value:
+                if isinstance(child, (list, tuple) ):
+                    # special case for Dict # FIXME
+                    _repr_tree(child[0], result, indent, _done, ids)
+                    _repr_tree(child[1], result, indent, _done, ids)
+                    result.append(indent + ',')
+                else:
+                    _repr_tree(child, result, indent, _done, ids)
+            result.append(  indent + "]" )
+        else:
+            result.append(  indent + field + " = " )
+            _repr_tree(value, result, indent, _done, ids)
+
+

pylibs/logilab/astng/brain/py2stdlib.py

@@ -0,0 +1,119 @@
+"""ASTNG hooks for the Python 2 standard library.
+
+Currently help understanding of :
+
+* hashlib.md5 and hashlib.sha1
+"""
+
+from logilab.astng import MANAGER
+from logilab.astng.builder import ASTNGBuilder
+
+MODULE_TRANSFORMS = {}
+
+def hashlib_transform(module):
+    fake = ASTNGBuilder(MANAGER).string_build('''
+
+class md5(object):
+  def __init__(self, value): pass
+  def hexdigest(self):
+    return u''
+
+class sha1(object):
+  def __init__(self, value): pass
+  def hexdigest(self):
+    return u''
+
+''')
+    for hashfunc in ('sha1', 'md5'):
+        module.locals[hashfunc] = fake.locals[hashfunc]
+
+def collections_transform(module):
+    fake = ASTNGBuilder(MANAGER).string_build('''
+
+class defaultdict(dict):
+    default_factory = None
+    def __missing__(self, key): pass
+
+class deque(object):
+    maxlen = 0
+    def __init__(iterable=None, maxlen=None): pass
+    def append(self, x): pass
+    def appendleft(self, x): pass
+    def clear(self): pass
+    def count(self, x): return 0
+    def extend(self, iterable): pass
+    def extendleft(self, iterable): pass
+    def pop(self): pass
+    def popleft(self): pass
+    def remove(self, value): pass
+    def reverse(self): pass
+    def rotate(self, n): pass
+
+''')
+
+    for klass in ('deque', 'defaultdict'):
+        module.locals[klass] = fake.locals[klass]
+
+def pkg_resources_transform(module):
+    fake = ASTNGBuilder(MANAGER).string_build('''
+
+def resource_exists(package_or_requirement, resource_name):
+    pass
+
+def resource_isdir(package_or_requirement, resource_name):
+    pass
+
+def resource_filename(package_or_requirement, resource_name):
+    pass
+
+def resource_stream(package_or_requirement, resource_name):
+    pass
+
+def resource_string(package_or_requirement, resource_name):
+    pass
+
+def resource_listdir(package_or_requirement, resource_name):
+    pass
+
+def extraction_error():
+    pass
+
+def get_cache_path(archive_name, names=()):
+    pass
+
+def postprocess(tempname, filename):
+    pass
+
+def set_extraction_path(path):
+    pass
+
+def cleanup_resources(force=False):
+    pass
+
+''')
+
+    for func_name, func in fake.locals.items():
+        module.locals[func_name] = func
+
+    # for func in ('resource_exists', 'resource_isdir', 'resource_filename',
+    #     'resource_stream', 'resource_string', 'resource_listdir',
+    #     'extraction_error', 'get_cache_path', 'postprocess',
+    #     'set_extraction_path', 'cleanup_resources'):
+
+    #     module.locals[func] = fake.locals[func]
+
+MODULE_TRANSFORMS['hashlib'] = hashlib_transform
+MODULE_TRANSFORMS['collections'] = collections_transform
+MODULE_TRANSFORMS['pkg_resources'] = pkg_resources_transform
+
+
+def transform(module):
+    try:
+        tr = MODULE_TRANSFORMS[module.name]
+    except KeyError:
+        pass
+    else:
+        tr(module)
+
+from logilab.astng import MANAGER
+MANAGER.register_transformer(transform)

pylibs/logilab/astng/builder.py

@@ -0,0 +1,226 @@
+# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+# copyright 2003-2010 Sylvain Thenault, all rights reserved.
+# contact mailto:thenault@gmail.com
+#
+# This file is part of logilab-astng.
+#
+# logilab-astng is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Lesser General Public License as published by the
+# Free Software Foundation, either version 2.1 of the License, or (at your
+# option) any later version.
+#
+# logilab-astng is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License
+# for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with logilab-astng. If not, see <http://www.gnu.org/licenses/>.
+"""The ASTNGBuilder makes astng from living object and / or from _ast
+
+The builder is not thread safe and can't be used to parse different sources
+at the same time.
+"""
+
+__docformat__ = "restructuredtext en"
+
+import sys, re
+from os.path import splitext, basename, dirname, exists, abspath
+
+from logilab.common.modutils import modpath_from_file
+
+from logilab.astng.exceptions import ASTNGBuildingException, InferenceError
+from logilab.astng.raw_building import InspectBuilder
+from logilab.astng.rebuilder import TreeRebuilder
+from logilab.astng.manager import ASTNGManager
+from logilab.astng.bases import YES, Instance
+
+from _ast import PyCF_ONLY_AST
+def parse(string):
+    return compile(string, "<string>", 'exec', PyCF_ONLY_AST)
+
+if sys.version_info >= (3, 0):
+    from tokenize import detect_encoding
+
+    def open_source_file(filename):
+        byte_stream = open(filename, 'bU')
+        encoding = detect_encoding(byte_stream.readline)[0]
+        stream = open(filename, 'U', encoding=encoding)
+        try:
+            data = stream.read()
+        except UnicodeError, uex: # wrong encodingg
+            # detect_encoding returns utf-8 if no encoding specified
+            msg = 'Wrong (%s) or no encoding specified' % encoding
+            raise ASTNGBuildingException(msg)
+        return stream, encoding, data
+
+else:
+    import re
+
+    _ENCODING_RGX = re.compile("\s*#+.*coding[:=]\s*([-\w.]+)")
+
+    def _guess_encoding(string):
+        """get encoding from a python file as string or return None if not found
+        """
+        # check for UTF-8 byte-order mark
+        if string.startswith('\xef\xbb\xbf'):
+            return 'UTF-8'
+        for line in string.split('\n', 2)[:2]:
+            # check for encoding declaration
+            match = _ENCODING_RGX.match(line)
+            if match is not None:
+                return match.group(1)
+
+    def open_source_file(filename):
+        """get data for parsing a file"""
+        stream = open(filename, 'U')
+        data = stream.read()
+        encoding = _guess_encoding(data)
+        return stream, encoding, data
+
+# ast NG builder ##############################################################
+
+MANAGER = ASTNGManager()
+
+class ASTNGBuilder(InspectBuilder):
+    """provide astng building methods"""
+    rebuilder = TreeRebuilder()
+
+    def __init__(self, manager=None):
+        self._manager = manager or MANAGER
+
+    def module_build(self, module, modname=None):
+        """build an astng from a living module instance
+        """
+        node = None
+        path = getattr(module, '__file__', None)
+        if path is not None:
+            path_, ext = splitext(module.__file__)
+            if ext in ('.py', '.pyc', '.pyo') and exists(path_ + '.py'):
+                node = self.file_build(path_ + '.py', modname)
+        if node is None:
+            # this is a built-in module
+            # get a partial representation by introspection
+            node = self.inspect_build(module, modname=modname, path=path)
+        return node
+
+    def file_build(self, path, modname=None):
+        """build astng from a source code file (i.e. from an ast)
+
+        path is expected to be a python source file
+        """
+        try:
+            stream, encoding, data = open_source_file(path)
+        except IOError, exc:
+            msg = 'Unable to load file %r (%s)' % (path, exc)
+            raise ASTNGBuildingException(msg)
+        except SyntaxError, exc: # py3k encoding specification error
+            raise ASTNGBuildingException(exc)
+        except LookupError, exc: # unknown encoding
+            raise ASTNGBuildingException(exc)
+        # get module name if necessary
+        if modname is None:
+            try:
+                modname = '.'.join(modpath_from_file(path))
+            except ImportError:
+                modname = splitext(basename(path))[0]
+        # build astng representation
+        node = self.string_build(data, modname, path)
+        node.file_encoding = encoding
+        return node
+
+    def string_build(self, data, modname='', path=None):
+        """build astng from source code string and return rebuilded astng"""
+        module = self._data_build(data, modname, path)
+        self._manager.astng_cache[module.name] = module
+        # post tree building steps after we stored the module in the cache:
+        for from_node in module._from_nodes:
+            self.add_from_names_to_locals(from_node)
+        # handle delayed assattr nodes
+        for delayed in module._delayed_assattr:
+            self.delayed_assattr(delayed)
+        if modname:
+            for transformer in self._manager.transformers:
+                transformer(module)
+        return module
+
+    def _data_build(self, data, modname, path):
+        """build tree node from data and add some informations"""
+        # this method could be wrapped with a pickle/cache function
+        node = parse(data + '\n')
+        if path is not None:
+            node_file = abspath(path)
+        else:
+            node_file = '<?>'
+        if modname.endswith('.__init__'):
+            modname = modname[:-9]
+            package = True
+        else:
+            package = path and path.find('__init__.py') > -1 or False
+        self.rebuilder.init()
+        module = self.rebuilder.visit_module(node, modname, package)
+        module.file = module.path = node_file
+        module._from_nodes = self.rebuilder._from_nodes
+        module._delayed_assattr = self.rebuilder._delayed_assattr
+        return module
+
+    def add_from_names_to_locals(self, node):
+        """store imported names to the locals;
+        resort the locals if coming from a delayed node
+        """
+
+        _key_func = lambda node: node.fromlineno
+        def sort_locals(my_list):
+            my_list.sort(key=_key_func)
+        for (name, asname) in node.names:
+            if name == '*':
+                try:
+                    imported = node.root().import_module(node.modname)
+                except ASTNGBuildingException:
+                    continue
+                for name in imported.wildcard_import_names():
+                    node.parent.set_local(name, node)
+                    sort_locals(node.parent.scope().locals[name])
+            else:
+                node.parent.set_local(asname or name, node)
+                sort_locals(node.parent.scope().locals[asname or name])
+
+    def delayed_assattr(self, node):
+        """visit a AssAttr node -> add name to locals, handle members
+        definition
+        """
+        try:
+            frame = node.frame()
+            for infered in node.expr.infer():
+                if infered is YES:
+                    continue
+                try:
+                    if infered.__class__ is Instance:
+                        infered = infered._proxied
+                        iattrs = infered.instance_attrs
+                    elif isinstance(infered, Instance):
+                        # Const, Tuple, ... we may be wrong, may be not, but
+                        # anyway we don't want to pollute builtin's namespace
+                        continue
+                    elif infered.is_function:
+                        iattrs = infered.instance_attrs
+                    else:
+                        iattrs = infered.locals
+                except AttributeError:
+                    # XXX log error
+                    #import traceback
+                    #traceback.print_exc()
+                    continue
+                values = iattrs.setdefault(node.attrname, [])
+                if node in values:
+                    continue
+                # get assign in __init__ first XXX useful ?
+                if frame.name == '__init__' and values and not \
+                       values[0].frame().name == '__init__':
+                    values.insert(0, node)
+                else:
+                    values.append(node)
+        except InferenceError:
+            pass
+

pylibs/logilab/astng/exceptions.py

@@ -0,0 +1,60 @@
+# This program is free software; you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by the Free Software
+# Foundation; either version 2 of the License, or (at your option) any later
+# version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc.,
+# 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+# copyright 2003-2010 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+# copyright 2003-2010 Sylvain Thenault, all rights reserved.
+# contact mailto:thenault@gmail.com
+#
+# This file is part of logilab-astng.
+#
+# logilab-astng is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Lesser General Public License as published by the
+# Free Software Foundation, either version 2.1 of the License, or (at your
+# option) any later version.
+#
+# logilab-astng is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License
+# for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with logilab-astng. If not, see <http://www.gnu.org/licenses/>.
+"""this module contains exceptions used in the astng library
+
+"""
+
+__doctype__ = "restructuredtext en"
+
+class ASTNGError(Exception):
+    """base exception class for all astng related exceptions"""
+
+class ASTNGBuildingException(ASTNGError):
+    """exception class when we are unable to build an astng representation"""
+
+class ResolveError(ASTNGError):
+    """base class of astng resolution/inference error"""
+
+class NotFoundError(ResolveError):
+    """raised when we are unable to resolve a name"""
+
+class InferenceError(ResolveError):
+    """raised when we are unable to infer a node"""
+
+class UnresolvableName(InferenceError):
+    """raised when we are unable to resolve a name"""
+
+class NoDefault(ASTNGError):
+    """raised by function's `default_value` method when an argument has
+    no default value
+    """
+

pylibs/logilab/astng/inference.py

@@ -0,0 +1,383 @@
+# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+# copyright 2003-2010 Sylvain Thenault, all rights reserved.
+# contact mailto:thenault@gmail.com
+#
+# This file is part of logilab-astng.
+#
+# logilab-astng is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Lesser General Public License as published by the
+# Free Software Foundation, either version 2.1 of the License, or (at your
+# option) any later version.
+#
+# logilab-astng is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License
+# for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with logilab-astng. If not, see <http://www.gnu.org/licenses/>.
+"""this module contains a set of functions to handle inference on astng trees
+"""
+
+__doctype__ = "restructuredtext en"
+
+from itertools import chain
+import sys
+
+from logilab.astng import nodes
+
+from logilab.astng.manager import ASTNGManager
+from logilab.astng.exceptions import (ASTNGBuildingException, ASTNGError,
+    InferenceError, NoDefault, NotFoundError, UnresolvableName)
+from logilab.astng.bases import YES, Instance, InferenceContext, Generator, \
+     _infer_stmts, copy_context, path_wrapper, raise_if_nothing_infered
+from logilab.astng.protocols import _arguments_infer_argname
+
+MANAGER = ASTNGManager()
+
+
+class CallContext:
+    """when inferring a function call, this class is used to remember values
+    given as argument
+    """
+    def __init__(self, args, starargs, dstarargs):
+        self.args = []
+        self.nargs = {}
+        for arg in args:
+            if isinstance(arg, nodes.Keyword):
+                self.nargs[arg.arg] = arg.value
+            else:
+                self.args.append(arg)
+        self.starargs = starargs
+        self.dstarargs = dstarargs
+
+    def infer_argument(self, funcnode, name, context):
+        """infer a function argument value according to the call context"""
+        # 1. search in named keywords
+        try:
+            return self.nargs[name].infer(context)
+        except KeyError:
+            # Function.args.args can be None in astng (means that we don't have
+            # information on argnames)
+            argindex = funcnode.args.find_argname(name)[0]
+            if argindex is not None:
+                # 2. first argument of instance/class method
+                if argindex == 0 and funcnode.type in ('method', 'classmethod'):
+                    if context.boundnode is not None:
+                        boundnode = context.boundnode
+                    else:
+                        # XXX can do better ?
+                        boundnode = funcnode.parent.frame()
+                    if funcnode.type == 'method':
+                        if not isinstance(boundnode, Instance):
+                            boundnode = Instance(boundnode)
+                        return iter((boundnode,))
+                    if funcnode.type == 'classmethod':
+                        return iter((boundnode,))
+                # 2. search arg index
+                try:
+                    return self.args[argindex].infer(context)
+                except IndexError:
+                    pass
+                # 3. search in *args (.starargs)
+                if self.starargs is not None:
+                    its = []
+                    for infered in self.starargs.infer(context):
+                        if infered is YES:
+                            its.append((YES,))
+                            continue
+                        try:
+                            its.append(infered.getitem(argindex, context).infer(context))
+                        except (InferenceError, AttributeError):
+                            its.append((YES,))
+                        except (IndexError, TypeError):
+                            continue
+                    if its:
+                        return chain(*its)
+        # 4. XXX search in **kwargs (.dstarargs)
+        if self.dstarargs is not None:
+            its = []
+            for infered in self.dstarargs.infer(context):
+                if infered is YES:
+                    its.append((YES,))
+                    continue
+                try:
+                    its.append(infered.getitem(name, context).infer(context))
+                except (InferenceError, AttributeError):
+                    its.append((YES,))
+                except (IndexError, TypeError):
+                    continue
+            if its:
+                return chain(*its)
+        # 5. */** argument, (Tuple or Dict)
+        if name == funcnode.args.vararg:
+            return iter((nodes.const_factory(())))
+        if name == funcnode.args.kwarg:
+            return iter((nodes.const_factory({})))
+        # 6. return default value if any
+        try:
+            return funcnode.args.default_value(name).infer(context)
+        except NoDefault:
+            raise InferenceError(name)
+
+
+# .infer method ###############################################################
+
+
+def infer_end(self, context=None):
+    """inference's end for node such as Module, Class, Function, Const...
+    """
+    yield self
+nodes.Module.infer = infer_end
+nodes.Class.infer = infer_end
+nodes.Function.infer = infer_end
+nodes.Lambda.infer = infer_end
+nodes.Const.infer = infer_end
+nodes.List.infer = infer_end
+nodes.Tuple.infer = infer_end
+nodes.Dict.infer = infer_end
+
+
+def infer_name(self, context=None):
+    """infer a Name: use name lookup rules"""
+    frame, stmts = self.lookup(self.name)
+    if not stmts:
+        raise UnresolvableName(self.name)
+    context = context.clone()
+    context.lookupname = self.name
+    return _infer_stmts(stmts, context, frame)
+nodes.Name.infer = path_wrapper(infer_name)
+nodes.AssName.infer_lhs = infer_name # won't work with a path wrapper
+
+
+def infer_callfunc(self, context=None):
+    """infer a CallFunc node by trying to guess what the function returns"""
+    callcontext = context.clone()
+    callcontext.callcontext = CallContext(self.args, self.starargs, self.kwargs)
+    callcontext.boundnode = None
+    for callee in self.func.infer(context):
+        if callee is YES:
+            yield callee
+            continue
+        try:
+            if hasattr(callee, 'infer_call_result'):
+                for infered in callee.infer_call_result(self, callcontext):
+                    yield infered
+        except InferenceError:
+            ## XXX log error ?
+            continue
+nodes.CallFunc.infer = path_wrapper(raise_if_nothing_infered(infer_callfunc))
+
+
+def infer_import(self, context=None, asname=True):
+    """infer an Import node: return the imported module/object"""
+    name = context.lookupname
+    if name is None:
+        raise InferenceError()
+    if asname:
+        yield self.do_import_module(self.real_name(name))
+    else:
+        yield self.do_import_module(name)
+nodes.Import.infer = path_wrapper(infer_import)
+
+def infer_name_module(self, name):
+    context = InferenceContext()
+    context.lookupname = name
+    return self.infer(context, asname=False)
+nodes.Import.infer_name_module = infer_name_module
+
+
+def infer_from(self, context=None, asname=True):
+    """infer a From nodes: return the imported module/object"""
+    name = context.lookupname
+    if name is None:
+        raise InferenceError()
+    if asname:
+        name = self.real_name(name)
+    module = self.do_import_module(self.modname)
+    try:
+        context = copy_context(context)
+        context.lookupname = name
+        return _infer_stmts(module.getattr(name, ignore_locals=module is self.root()), context)
+    except NotFoundError:
+        raise InferenceError(name)
+nodes.From.infer = path_wrapper(infer_from)
+
+
+def infer_getattr(self, context=None):
+    """infer a Getattr node by using getattr on the associated object"""
+    #context = context.clone()
+    for owner in self.expr.infer(context):
+        if owner is YES:
+            yield owner
+            continue
+        try:
+            context.boundnode = owner
+            for obj in owner.igetattr(self.attrname, context):
+                yield obj
+            context.boundnode = None
+        except (NotFoundError, InferenceError):
+            context.boundnode = None
+        except AttributeError:
+            # XXX method / function
+            context.boundnode = None
+nodes.Getattr.infer = path_wrapper(raise_if_nothing_infered(infer_getattr))
+nodes.AssAttr.infer_lhs = raise_if_nothing_infered(infer_getattr) # # won't work with a path wrapper
+
+
+def infer_global(self, context=None):
+    if context.lookupname is None:
+        raise InferenceError()
+    try:
+        return _infer_stmts(self.root().getattr(context.lookupname), context)
+    except NotFoundError:
+        raise InferenceError()
+nodes.Global.infer = path_wrapper(infer_global)
+
+
+def infer_subscript(self, context=None):
+    """infer simple subscription such as [1,2,3][0] or (1,2,3)[-1]"""
+    if isinstance(self.slice, nodes.Index):
+        index = self.slice.value.infer(context).next()
+        if index is YES:
+            yield YES
+            return
+        try:
+            # suppose it's a Tuple/List node (attribute error else)
+            # XXX infer self.value?
+            assigned = self.value.getitem(index.value, context)
+        except AttributeError:
+            raise InferenceError()
+        except (IndexError, TypeError):
+            yield YES
+            return
+        for infered in assigned.infer(context):
+            yield infered
+    else:
+        raise InferenceError()
+nodes.Subscript.infer = path_wrapper(infer_subscript)
+nodes.Subscript.infer_lhs = raise_if_nothing_infered(infer_subscript)
+
+
+UNARY_OP_METHOD = {'+': '__pos__',
+                   '-': '__neg__',
+                   '~': '__invert__',
+                   'not': None, # XXX not '__nonzero__'
+                  }
+
+def infer_unaryop(self, context=None):
+    for operand in self.operand.infer(context):
+        try:
+            yield operand.infer_unary_op(self.op)
+        except TypeError:
+            continue
+        except AttributeError:
+            meth = UNARY_OP_METHOD[self.op]
+            if meth is None:
+                yield YES
+            else:
+                try:
+                    # XXX just suppose if the type implement meth, returned type
+                    # will be the same
+                    operand.getattr(meth)
+                    yield operand
+                except GeneratorExit:
+                    raise
+                except:
+                    yield YES
+nodes.UnaryOp.infer = path_wrapper(infer_unaryop)
+
+
+BIN_OP_METHOD = {'+':  '__add__',
+                 '-':  '__sub__',
+                 '/':  '__div__',
+                 '//': '__floordiv__',
+                 '*':  '__mul__',
+                 '**': '__power__',
+                 '%':  '__mod__',
+                 '&':  '__and__',
+                 '|':  '__or__',
+                 '^':  '__xor__',
+                 '<<': '__lshift__',
+                 '>>': '__rshift__',
+                 }
+
+def _infer_binop(operator, operand1, operand2, context, failures=None):
+    if operand1 is YES:
+        yield operand1
+        return
+    try:
+        for valnode in operand1.infer_binary_op(operator, operand2, context):
+            yield valnode
+    except AttributeError:
+        try:
+            # XXX just suppose if the type implement meth, returned type
+            # will be the same
+            operand1.getattr(BIN_OP_METHOD[operator])
+            yield operand1
+        except:
+            if failures is None:
+                yield YES
+            else:
+                failures.append(operand1)
+
+def infer_binop(self, context=None):
+    failures = []
+    for lhs in self.left.infer(context):
+        for val in _infer_binop(self.op, lhs, self.right, context, failures):
+            yield val
+    for lhs in failures:
+        for rhs in self.right.infer(context):
+            for val in _infer_binop(self.op, rhs, lhs, context):
+                yield val
+nodes.BinOp.infer = path_wrapper(infer_binop)
+
+
+def infer_arguments(self, context=None):
+    name = context.lookupname
+    if name is None:
+        raise InferenceError()
+    return _arguments_infer_argname(self, name, context)
+nodes.Arguments.infer = infer_arguments
+
+
+def infer_ass(self, context=None):
+    """infer a AssName/AssAttr: need to inspect the RHS part of the
+    assign node
+    """
+    stmt = self.statement()
+    if isinstance(stmt, nodes.AugAssign):
+        return stmt.infer(context)
+    stmts = list(self.assigned_stmts(context=context))
+    return _infer_stmts(stmts, context)
+nodes.AssName.infer = path_wrapper(infer_ass)
+nodes.AssAttr.infer = path_wrapper(infer_ass)
+
+def infer_augassign(self, context=None):
+    failures = []
+    for lhs in self.target.infer_lhs(context):
+        for val in _infer_binop(self.op, lhs, self.value, context, failures):
+            yield val
+    for lhs in failures:
+        for rhs in self.value.infer(context):
+            for val in _infer_binop(self.op, rhs, lhs, context):
+                yield val
+nodes.AugAssign.infer = path_wrapper(infer_augassign)
+
+
+# no infer method on DelName and DelAttr (expected InferenceError)
+
+
+def infer_empty_node(self, context=None):
+    if not self.has_underlying_object():
+        yield YES
+    else:
+        try:
+            for infered in MANAGER.infer_astng_from_something(self.object,
+                                                              context=context):
+                yield infered
+        except ASTNGError:
+            yield YES
+nodes.EmptyNode.infer = path_wrapper(infer_empty_node)
+

pylibs/logilab/astng/inspector.py

@@ -0,0 +1,289 @@
+# This program is free software; you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by the Free Software
+# Foundation; either version 2 of the License, or (at your option) any later
+# version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc.,
+# 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+# copyright 2003-2010 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+# copyright 2003-2010 Sylvain Thenault, all rights reserved.
+# contact mailto:thenault@gmail.com
+#
+# This file is part of logilab-astng.
+#
+# logilab-astng is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Lesser General Public License as published by the
+# Free Software Foundation, either version 2.1 of the License, or (at your
+# option) any later version.
+#
+# logilab-astng is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License
+# for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with logilab-astng. If not, see <http://www.gnu.org/licenses/>.
+"""visitor doing some postprocessing on the astng tree.
+Try to resolve definitions (namespace) dictionary, relationship...
+
+This module has been imported from pyreverse
+"""
+
+__docformat__ = "restructuredtext en"
+
+from os.path import dirname
+
+from logilab.common.modutils import get_module_part, is_relative, \
+     is_standard_module
+
+from logilab import astng
+from logilab.astng.exceptions import InferenceError
+from logilab.astng.utils import LocalsVisitor
+
+class IdGeneratorMixIn:
+    """
+    Mixin adding the ability to generate integer uid
+    """
+    def __init__(self, start_value=0):
+        self.id_count = start_value
+    
+    def init_counter(self, start_value=0):
+        """init the id counter
+        """
+        self.id_count = start_value
+        
+    def generate_id(self):
+        """generate a new identifier
+        """
+        self.id_count += 1
+        return self.id_count
+
+
+class Linker(IdGeneratorMixIn, LocalsVisitor):
+    """
+    walk on the project tree and resolve relationships.
+    
+    According to options the following attributes may be added to visited nodes:
+    
+    * uid,
+      a unique identifier for the node (on astng.Project, astng.Module,
+      astng.Class and astng.locals_type). Only if the linker has been instantiated
+      with tag=True parameter (False by default).
+            
+    * Function
+      a mapping from locals names to their bounded value, which may be a
+      constant like a string or an integer, or an astng node (on astng.Module,
+      astng.Class and astng.Function).
+
+    * instance_attrs_type
+      as locals_type but for klass member attributes (only on astng.Class)
+      
+    * implements,
+      list of implemented interface _objects_ (only on astng.Class nodes)
+    """
+    
+    def __init__(self, project, inherited_interfaces=0, tag=False):
+        IdGeneratorMixIn.__init__(self)
+        LocalsVisitor.__init__(self)
+        # take inherited interface in consideration or not
+        self.inherited_interfaces = inherited_interfaces
+        # tag nodes or not
+        self.tag = tag
+        # visited project
+        self.project = project
+
+        
+    def visit_project(self, node):
+        """visit an astng.Project node
+        
+         * optionally tag the node with a unique id
+        """
+        if self.tag:
+            node.uid = self.generate_id()
+        for module in node.modules:
+            self.visit(module)
+            
+    def visit_package(self, node):
+        """visit an astng.Package node
+        
+         * optionally tag the node with a unique id
+        """
+        if self.tag:
+            node.uid = self.generate_id()
+        for subelmt in node.values():
+            self.visit(subelmt)
+            
+    def visit_module(self, node):
+        """visit an astng.Module node
+        
+         * set the locals_type mapping
+         * set the depends mapping
+         * optionally tag the node with a unique id
+        """
+        if hasattr(node, 'locals_type'):
+            return
+        node.locals_type = {}
+        node.depends = []
+        if self.tag:
+            node.uid = self.generate_id()
+    
+    def visit_class(self, node):
+        """visit an astng.Class node
+        
+         * set the locals_type and instance_attrs_type mappings
+         * set the implements list and build it
+         * optionally tag the node with a unique id
+        """
+        if hasattr(node, 'locals_type'):
+            return
+        node.locals_type = {}
+        if self.tag:
+            node.uid = self.generate_id()
+        # resolve ancestors
+        for baseobj in node.ancestors(recurs=False):
+            specializations = getattr(baseobj, 'specializations', [])
+            specializations.append(node)
+            baseobj.specializations = specializations
+        # resolve instance attributes
+        node.instance_attrs_type = {}
+        for assattrs in node.instance_attrs.values():
+            for assattr in assattrs:
+                self.handle_assattr_type(assattr, node)
+        # resolve implemented interface
+        try:
+            node.implements = list(node.interfaces(self.inherited_interfaces))
+        except InferenceError:
+            node.implements = ()
+
+    def visit_function(self, node):
+        """visit an astng.Function node
+        
+         * set the locals_type mapping
+         * optionally tag the node with a unique id
+        """
+        if hasattr(node, 'locals_type'):
+            return
+        node.locals_type = {}
+        if self.tag:
+            node.uid = self.generate_id()
+            
+    link_project = visit_project
+    link_module = visit_module
+    link_class = visit_class
+    link_function = visit_function
+        
+    def visit_assname(self, node):
+        """visit an astng.AssName node
+
+        handle locals_type
+        """
+        # avoid double parsing done by different Linkers.visit
+        # running over the same project:
+        if hasattr(node, '_handled'):
+            return
+        node._handled = True
+        if node.name in node.frame():
+            frame = node.frame()
+        else:
+            # the name has been defined as 'global' in the frame and belongs
+            # there. Btw the frame is not yet visited as the name is in the 
+            # root locals; the frame hence has no locals_type attribute
+            frame = node.root()
+        try:
+            values = node.infered()
+            try:
+                already_infered = frame.locals_type[node.name]
+                for valnode in values:
+                    if not valnode in already_infered:
+                        already_infered.append(valnode)
+            except KeyError:
+                frame.locals_type[node.name] = values
+        except astng.InferenceError:
+            pass
+
+    def handle_assattr_type(self, node, parent):
+        """handle an astng.AssAttr node
+
+        handle instance_attrs_type
+        """
+        try:
+            values = list(node.infer())
+            try:
+                already_infered = parent.instance_attrs_type[node.attrname]
+                for valnode in values:
+                    if not valnode in already_infered:
+                        already_infered.append(valnode)
+            except KeyError:
+                parent.instance_attrs_type[node.attrname] = values
+        except astng.InferenceError:
+            pass
+            
+    def visit_import(self, node):
+        """visit an astng.Import node
+        
+        resolve module dependencies
+        """
+        context_file = node.root().file
+        for name in node.names:
+            relative = is_relative(name[0], context_file)
+            self._imported_module(node, name[0], relative)
+        
+
+    def visit_from(self, node):
+        """visit an astng.From node
+        
+        resolve module dependencies
+        """
+        basename = node.modname
+        context_file = node.root().file
+        if context_file is not None:
+            relative = is_relative(basename, context_file)
+        else:
+            relative = False
+        for name in node.names:
+            if name[0] == '*':
+                continue
+            # analyze dependencies
+            fullname = '%s.%s' % (basename, name[0])
+            if fullname.find('.') > -1:
+                try:
+                    # XXX: don't use get_module_part, missing package precedence
+                    fullname = get_module_part(fullname)
+                except ImportError:
+                    continue
+            if fullname != basename:
+                self._imported_module(node, fullname, relative)
+
+        
+    def compute_module(self, context_name, mod_path):
+        """return true if the module should be added to dependencies"""
+        package_dir = dirname(self.project.path)
+        if context_name == mod_path:
+            return 0
+        elif is_standard_module(mod_path, (package_dir,)):
+            return 1
+        return 0
+    
+    # protected methods ########################################################
+
+    def _imported_module(self, node, mod_path, relative):
+        """notify an imported module, used to analyze dependencies
+        """
+        module = node.root()
+        context_name = module.name
+        if relative:
+            mod_path = '%s.%s' % ('.'.join(context_name.split('.')[:-1]),
+                                  mod_path)
+        if self.compute_module(context_name, mod_path):
+            # handle dependencies
+            if not hasattr(module, 'depends'):
+                module.depends = []
+            mod_paths = module.depends
+            if not mod_path in mod_paths:
+                mod_paths.append(mod_path)

pylibs/logilab/astng/manager.py

@@ -0,0 +1,299 @@
+# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+# copyright 2003-2010 Sylvain Thenault, all rights reserved.
+# contact mailto:thenault@gmail.com
+#
+# This file is part of logilab-astng.
+#
+# logilab-astng is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Lesser General Public License as published by the
+# Free Software Foundation, either version 2.1 of the License, or (at your
+# option) any later version.
+#
+# logilab-astng is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License
+# for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with logilab-astng. If not, see <http://www.gnu.org/licenses/>.
+"""astng manager: avoid multiple astng build of a same module when
+possible by providing a class responsible to get astng representation
+from various source and using a cache of built modules)
+"""
+
+__docformat__ = "restructuredtext en"
+
+import sys
+import os
+from os.path import dirname, basename, abspath, join, isdir, exists
+
+from logilab.common.modutils import NoSourceFile, is_python_source, \
+     file_from_modpath, load_module_from_name, modpath_from_file, \
+     get_module_files, get_source_file, zipimport
+from logilab.common.configuration import OptionsProviderMixIn
+
+from logilab.astng.exceptions import ASTNGBuildingException
+
+def astng_wrapper(func, modname):
+    """wrapper to give to ASTNGManager.project_from_files"""
+    print 'parsing %s...' % modname
+    try:
+        return func(modname)
+    except ASTNGBuildingException, exc:
+        print exc
+    except Exception, exc:
+        import traceback
+        traceback.print_exc()
+
+def _silent_no_wrap(func, modname):
+    """silent wrapper that doesn't do anything; can be used for tests"""
+    return func(modname)
+
+def safe_repr(obj):
+    try:
+        return repr(obj)
+    except:
+        return '???'
+
+
+
+class ASTNGManager(OptionsProviderMixIn):
+    """the astng manager, responsible to build astng from files
+     or modules.
+
+    Use the Borg pattern.
+    """
+
+    name = 'astng loader'
+    options = (("ignore",
+                {'type' : "csv", 'metavar' : "<file>",
+                 'dest' : "black_list", "default" : ('CVS',),
+                 'help' : "add <file> (may be a directory) to the black list\
+. It should be a base name, not a path. You may set this option multiple times\
+."}),
+               ("project",
+                {'default': "No Name", 'type' : 'string', 'short': 'p',
+                 'metavar' : '<project name>',
+                 'help' : 'set the project name.'}),
+               )
+    brain = {}
+    def __init__(self):
+        self.__dict__ = ASTNGManager.brain
+        if not self.__dict__:
+            OptionsProviderMixIn.__init__(self)
+            self.load_defaults()
+            # NOTE: cache entries are added by the [re]builder
+            self.astng_cache = {}
+            self._mod_file_cache = {}
+            self.transformers = []
+
+    def astng_from_file(self, filepath, modname=None, fallback=True, source=False):
+        """given a module name, return the astng object"""
+        try:
+            filepath = get_source_file(filepath, include_no_ext=True)
+            source = True
+        except NoSourceFile:
+            pass
+        if modname is None:
+            try:
+                modname = '.'.join(modpath_from_file(filepath))
+            except ImportError:
+                modname = filepath
+        if modname in self.astng_cache:
+            return self.astng_cache[modname]
+        if source:
+            from logilab.astng.builder import ASTNGBuilder
+            return ASTNGBuilder(self).file_build(filepath, modname)
+        elif fallback and modname:
+            return self.astng_from_module_name(modname)
+        raise ASTNGBuildingException('unable to get astng for file %s' %
+                                     filepath)
+
+    def astng_from_module_name(self, modname, context_file=None):
+        """given a module name, return the astng object"""
+        if modname in self.astng_cache:
+            return self.astng_cache[modname]
+        if modname == '__main__':
+            from logilab.astng.builder import ASTNGBuilder
+            return ASTNGBuilder(self).string_build('', modname)
+        old_cwd = os.getcwd()
+        if context_file:
+            os.chdir(dirname(context_file))
+        try:
+            filepath = self.file_from_module_name(modname, context_file)
+            if filepath is not None and not is_python_source(filepath):
+                module = self.zip_import_data(filepath)
+                if module is not None:
+                    return module
+            if filepath is None or not is_python_source(filepath):
+                try:
+                    module = load_module_from_name(modname)
+                except Exception, ex:
+                    msg = 'Unable to load module %s (%s)' % (modname, ex)
+                    raise ASTNGBuildingException(msg)
+                return self.astng_from_module(module, modname)
+            return self.astng_from_file(filepath, modname, fallback=False)
+        finally:
+            os.chdir(old_cwd)
+
+    def zip_import_data(self, filepath):
+        if zipimport is None:
+            return None
+        from logilab.astng.builder import ASTNGBuilder
+        builder = ASTNGBuilder(self)
+        for ext in ('.zip', '.egg'):
+            try:
+                eggpath, resource = filepath.rsplit(ext + '/', 1)
+            except ValueError:
+                continue
+            try:
+                importer = zipimport.zipimporter(eggpath + ext)
+                zmodname = resource.replace('/', '.')
+                if importer.is_package(resource):
+                    zmodname =  zmodname + '.__init__'
+                module = builder.string_build(importer.get_source(resource),
+                                              zmodname, filepath)
+                return module
+            except:
+                continue
+        return None
+
+    def file_from_module_name(self, modname, contextfile):
+        try:
+            value = self._mod_file_cache[(modname, contextfile)]
+        except KeyError:
+            try:
+                value = file_from_modpath(modname.split('.'),
+                                          context_file=contextfile)
+            except ImportError, ex:
+                msg = 'Unable to load module %s (%s)' % (modname, ex)
+                value = ASTNGBuildingException(msg)
+            self._mod_file_cache[(modname, contextfile)] = value
+        if isinstance(value, ASTNGBuildingException):
+            raise value
+        return value
+
+    def astng_from_module(self, module, modname=None):
+        """given an imported module, return the astng object"""
+        modname = modname or module.__name__
+        if modname in self.astng_cache:
+            return self.astng_cache[modname]
+        try:
+            # some builtin modules don't have __file__ attribute
+            filepath = module.__file__
+            if is_python_source(filepath):
+                return self.astng_from_file(filepath, modname)
+        except AttributeError:
+            pass
+        from logilab.astng.builder import ASTNGBuilder
+        return ASTNGBuilder(self).module_build(module, modname)
+
+    def astng_from_class(self, klass, modname=None):
+        """get astng for the given class"""
+        if modname is None:
+            try:
+                modname = klass.__module__
+            except AttributeError:
+                raise ASTNGBuildingException(
+                    'Unable to get module for class %s' % safe_repr(klass))
+        modastng = self.astng_from_module_name(modname)
+        return modastng.getattr(klass.__name__)[0] # XXX
+
+
+    def infer_astng_from_something(self, obj, context=None):
+        """infer astng for the given class"""
+        if hasattr(obj, '__class__') and not isinstance(obj, type):
+            klass = obj.__class__
+        else:
+            klass = obj
+        try:
+            modname = klass.__module__
+        except AttributeError:
+            raise ASTNGBuildingException(
+                'Unable to get module for %s' % safe_repr(klass))
+        except Exception, ex:
+            raise ASTNGBuildingException(
+                'Unexpected error while retrieving module for %s: %s'
+                % (safe_repr(klass), ex))
+        try:
+            name = klass.__name__
+        except AttributeError:
+            raise ASTNGBuildingException(
+                'Unable to get name for %s' % safe_repr(klass))
+        except Exception, ex:
+            raise ASTNGBuildingException(
+                'Unexpected error while retrieving name for %s: %s'
+                % (safe_repr(klass), ex))
+        # take care, on living object __module__ is regularly wrong :(
+        modastng = self.astng_from_module_name(modname)
+        if klass is obj:
+            for  infered in modastng.igetattr(name, context):
+                yield infered
+        else:
+            for infered in modastng.igetattr(name, context):
+                yield infered.instanciate_class()
+
+    def project_from_files(self, files, func_wrapper=astng_wrapper,
+                           project_name=None, black_list=None):
+        """return a Project from a list of files or modules"""
+        # build the project representation
+        project_name = project_name or self.config.project
+        black_list = black_list or self.config.black_list
+        project = Project(project_name)
+        for something in files:
+            if not exists(something):
+                fpath = file_from_modpath(something.split('.'))
+            elif isdir(something):
+                fpath = join(something, '__init__.py')
+            else:
+                fpath = something
+            astng = func_wrapper(self.astng_from_file, fpath)
+            if astng is None:
+                continue
+            # XXX why is first file defining the project.path ?
+            project.path = project.path or astng.file
+            project.add_module(astng)
+            base_name = astng.name
+            # recurse in package except if __init__ was explicitly given
+            if astng.package and something.find('__init__') == -1:
+                # recurse on others packages / modules if this is a package
+                for fpath in get_module_files(dirname(astng.file),
+                                              black_list):
+                    astng = func_wrapper(self.astng_from_file, fpath)
+                    if astng is None or astng.name == base_name:
+                        continue
+                    project.add_module(astng)
+        return project
+
+    def register_transformer(self, transformer):
+        self.transformers.append(transformer)
+
+class Project:
+    """a project handle a set of modules / packages"""
+    def __init__(self, name=''):
+        self.name = name
+        self.path = None
+        self.modules = []
+        self.locals = {}
+        self.__getitem__ = self.locals.__getitem__
+        self.__iter__ = self.locals.__iter__
+        self.values = self.locals.values
+        self.keys = self.locals.keys
+        self.items = self.locals.items
+
+    def add_module(self, node):
+        self.locals[node.name] = node
+        self.modules.append(node)
+
+    def get_module(self, name):
+        return self.locals[name]
+
+    def get_children(self):
+        return self.modules
+
+    def __repr__(self):
+        return '<Project %r at %s (%s modules)>' % (self.name, id(self),
+                                                    len(self.modules))
+
+

pylibs/logilab/astng/mixins.py

@@ -0,0 +1,136 @@
+# This program is free software; you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by the Free Software
+# Foundation; either version 2 of the License, or (at your option) any later
+# version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc.,
+# 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+# copyright 2003-2010 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+# copyright 2003-2010 Sylvain Thenault, all rights reserved.
+# contact mailto:thenault@gmail.com
+#
+# This file is part of logilab-astng.
+#
+# logilab-astng is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Lesser General Public License as published by the
+# Free Software Foundation, either version 2.1 of the License, or (at your
+# option) any later version.
+#
+# logilab-astng is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License
+# for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with logilab-astng. If not, see <http://www.gnu.org/licenses/>.
+"""This module contains some mixins for the different nodes.
+"""
+
+from logilab.astng.exceptions import (ASTNGBuildingException, InferenceError,
+                                      NotFoundError)
+
+
+class BlockRangeMixIn(object):
+    """override block range """
+    def set_line_info(self, lastchild):
+        self.fromlineno = self.lineno
+        self.tolineno = lastchild.tolineno
+        self.blockstart_tolineno = self._blockstart_toline()
+
+    def _elsed_block_range(self, lineno, orelse, last=None):
+        """handle block line numbers range for try/finally, for, if and while
+        statements
+        """
+        if lineno == self.fromlineno:
+            return lineno, lineno
+        if orelse:
+            if lineno >= orelse[0].fromlineno:
+                return lineno, orelse[-1].tolineno
+            return lineno, orelse[0].fromlineno - 1
+        return lineno, last or self.tolineno
+
+class FilterStmtsMixin(object):
+    """Mixin for statement filtering and assignment type"""
+
+    def _get_filtered_stmts(self, _, node, _stmts, mystmt):
+        """method used in _filter_stmts to get statemtents and trigger break"""
+        if self.statement() is mystmt:
+            # original node's statement is the assignment, only keep
+            # current node (gen exp, list comp)
+            return [node], True
+        return _stmts, False
+
+    def ass_type(self):
+        return self
+
+
+class AssignTypeMixin(object):
+
+    def ass_type(self):
+        return self
+
+    def _get_filtered_stmts(self, lookup_node, node, _stmts, mystmt):
+        """method used in filter_stmts"""
+        if self is mystmt:
+            return _stmts, True
+        if self.statement() is mystmt:
+            # original node's statement is the assignment, only keep
+            # current node (gen exp, list comp)
+            return [node], True
+        return _stmts, False
+
+
+class ParentAssignTypeMixin(AssignTypeMixin):
+
+    def ass_type(self):
+        return self.parent.ass_type()
+
+
+
+class FromImportMixIn(FilterStmtsMixin):
+    """MixIn for From and Import Nodes"""
+
+    def _infer_name(self, frame, name):
+        return name
+
+    def do_import_module(self, modname):
+        """return the ast for a module whose name is <modname> imported by <self>
+        """
+        # handle special case where we are on a package node importing a module
+        # using the same name as the package, which may end in an infinite loop
+        # on relative imports
+        # XXX: no more needed ?
+        mymodule = self.root()
+        level = getattr(self, 'level', None) # Import as no level
+        # XXX we should investigate deeper if we really want to check
+        # importing itself: modname and mymodule.name be relative or absolute
+        if mymodule.relative_to_absolute_name(modname, level) == mymodule.name:
+            # FIXME: we used to raise InferenceError here, but why ?
+            return mymodule
+        try:
+            return mymodule.import_module(modname, level=level)
+        except ASTNGBuildingException:
+            raise InferenceError(modname)
+        except SyntaxError, ex:
+            raise InferenceError(str(ex))
+
+    def real_name(self, asname):
+        """get name from 'as' name"""
+        for name, _asname in self.names:
+            if name == '*':
+                return asname
+            if not _asname:
+                name = name.split('.', 1)[0]
+                _asname = name
+            if asname == _asname:
+                return name
+        raise NotFoundError(asname)
+
+
+

pylibs/logilab/astng/node_classes.py

@@ -0,0 +1,903 @@
+# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+# copyright 2003-2010 Sylvain Thenault, all rights reserved.
+# contact mailto:thenault@gmail.com
+#
+# This file is part of logilab-astng.
+#
+# logilab-astng is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Lesser General Public License as published by the
+# Free Software Foundation, either version 2.1 of the License, or (at your
+# option) any later version.
+#
+# logilab-astng is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License
+# for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with logilab-astng. If not, see <http://www.gnu.org/licenses/>.
+"""Module for some node classes. More nodes in scoped_nodes.py
+"""
+
+import sys
+
+from logilab.astng import BUILTINS_MODULE
+from logilab.astng.exceptions import NoDefault
+from logilab.astng.bases import (NodeNG, Statement, Instance, InferenceContext,
+                                 _infer_stmts, YES)
+from logilab.astng.mixins import BlockRangeMixIn, AssignTypeMixin, \
+                                 ParentAssignTypeMixin, FromImportMixIn
+
+
+def unpack_infer(stmt, context=None):
+    """recursively generate nodes inferred by the given statement.
+    If the inferred value is a list or a tuple, recurse on the elements
+    """
+    if isinstance(stmt, (List, Tuple)):
+        for elt in stmt.elts:
+            for infered_elt in unpack_infer(elt, context):
+                yield infered_elt
+        return
+    infered = stmt.infer(context).next()
+    if infered is stmt or infered is YES:
+        yield infered
+        return
+    for infered in stmt.infer(context):
+        for inf_inf in unpack_infer(infered, context):
+            yield inf_inf
+
+
+def are_exclusive(stmt1, stmt2, exceptions=None):
+    """return true if the two given statements are mutually exclusive
+
+    `exceptions` may be a list of exception names. If specified, discard If
+    branches and check one of the statement is in an exception handler catching
+    one of the given exceptions.
+
+    algorithm :
+     1) index stmt1's parents
+     2) climb among stmt2's parents until we find a common parent
+     3) if the common parent is a If or TryExcept statement, look if nodes are
+        in exclusive branches
+    """
+    # index stmt1's parents
+    stmt1_parents = {}
+    children = {}
+    node = stmt1.parent
+    previous = stmt1
+    while node:
+        stmt1_parents[node] = 1
+        children[node] = previous
+        previous = node
+        node = node.parent
+    # climb among stmt2's parents until we find a common parent
+    node = stmt2.parent
+    previous = stmt2
+    while node:
+        if node in stmt1_parents:
+            # if the common parent is a If or TryExcept statement, look if
+            # nodes are in exclusive branches
+            if isinstance(node, If) and exceptions is None:
+                if (node.locate_child(previous)[1]
+                    is not node.locate_child(children[node])[1]):
+                    return True
+            elif isinstance(node, TryExcept):
+                c2attr, c2node = node.locate_child(previous)
+                c1attr, c1node = node.locate_child(children[node])
+                if c1node is not c2node:
+                    if ((c2attr == 'body' and c1attr == 'handlers' and children[node].catch(exceptions)) or
+                        (c2attr == 'handlers' and c1attr == 'body' and previous.catch(exceptions)) or
+                        (c2attr == 'handlers' and c1attr == 'orelse') or
+                        (c2attr == 'orelse' and c1attr == 'handlers')):
+                        return True
+                elif c2attr == 'handlers' and c1attr == 'handlers':
+                    return previous is not children[node]
+            return False
+        previous = node
+        node = node.parent
+    return False
+
+
+class LookupMixIn(object):
+    """Mixin looking up a name in the right scope
+    """
+
+    def lookup(self, name):
+        """lookup a variable name
+
+        return the scope node and the list of assignments associated to the given
+        name according to the scope where it has been found (locals, globals or
+        builtin)
+
+        The lookup is starting from self's scope. If self is not a frame itself and
+        the name is found in the inner frame locals, statements will be filtered
+        to remove ignorable statements according to self's location
+        """
+        return self.scope().scope_lookup(self, name)
+
+    def ilookup(self, name):
+        """infered lookup
+
+        return an iterator on infered values of the statements returned by
+        the lookup method
+        """
+        frame, stmts = self.lookup(name)
+        context = InferenceContext()
+        return _infer_stmts(stmts, context, frame)
+
+    def _filter_stmts(self, stmts, frame, offset):
+        """filter statements to remove ignorable statements.
+
+        If self is not a frame itself and the name is found in the inner
+        frame locals, statements will be filtered to remove ignorable
+        statements according to self's location
+        """
+        # if offset == -1, my actual frame is not the inner frame but its parent
+        #
+        # class A(B): pass
+        #
+        # we need this to resolve B correctly
+        if offset == -1:
+            myframe = self.frame().parent.frame()
+        else:
+            myframe = self.frame()
+        if not myframe is frame or self is frame:
+            return stmts
+        mystmt = self.statement()
+        # line filtering if we are in the same frame
+        #
+        # take care node may be missing lineno information (this is the case for
+        # nodes inserted for living objects)
+        if myframe is frame and mystmt.fromlineno is not None:
+            assert mystmt.fromlineno is not None, mystmt
+            mylineno = mystmt.fromlineno + offset
+        else:
+            # disabling lineno filtering
+            mylineno = 0
+        _stmts = []
+        _stmt_parents = []
+        for node in stmts:
+            stmt = node.statement()
+            # line filtering is on and we have reached our location, break
+            if mylineno > 0 and stmt.fromlineno > mylineno:
+                break
+            assert hasattr(node, 'ass_type'), (node, node.scope(),
+                                               node.scope().locals)
+            ass_type = node.ass_type()
+
+            if node.has_base(self):
+                break
+
+            _stmts, done = ass_type._get_filtered_stmts(self, node, _stmts, mystmt)
+            if done:
+                break
+
+            optional_assign = ass_type.optional_assign
+            if optional_assign and ass_type.parent_of(self):
+                # we are inside a loop, loop var assigment is hidding previous
+                # assigment
+                _stmts = [node]
+                _stmt_parents = [stmt.parent]
+                continue
+
+            # XXX comment various branches below!!!
+            try:
+                pindex = _stmt_parents.index(stmt.parent)
+            except ValueError:
+                pass
+            else:
+                # we got a parent index, this means the currently visited node
+                # is at the same block level as a previously visited node
+                if _stmts[pindex].ass_type().parent_of(ass_type):
+                    # both statements are not at the same block level
+                    continue
+                # if currently visited node is following previously considered
+                # assignement and both are not exclusive, we can drop the
+                # previous one. For instance in the following code ::
+                #
+                #   if a:
+                #     x = 1
+                #   else:
+                #     x = 2
+                #   print x
+                #
+                # we can't remove neither x = 1 nor x = 2 when looking for 'x'
+                # of 'print x'; while in the following ::
+                #
+                #   x = 1
+                #   x = 2
+                #   print x
+                #
+                # we can remove x = 1 when we see x = 2
+                #
+                # moreover, on loop assignment types, assignment won't
+                # necessarily be done if the loop has no iteration, so we don't
+                # want to clear previous assigments if any (hence the test on
+                # optional_assign)
+                if not (optional_assign or are_exclusive(_stmts[pindex], node)):
+                    del _stmt_parents[pindex]
+                    del _stmts[pindex]
+            if isinstance(node, AssName):
+                if not optional_assign and stmt.parent is mystmt.parent:
+                    _stmts = []
+                    _stmt_parents = []
+            elif isinstance(node, DelName):
+                _stmts = []
+                _stmt_parents = []
+                continue
+            if not are_exclusive(self, node):
+                _stmts.append(node)
+                _stmt_parents.append(stmt.parent)
+        return _stmts
+
+# Name classes
+
+class AssName(LookupMixIn, ParentAssignTypeMixin, NodeNG):
+    """class representing an AssName node"""
+
+
+class DelName(LookupMixIn, ParentAssignTypeMixin, NodeNG):
+    """class representing a DelName node"""
+
+
+class Name(LookupMixIn, NodeNG):
+    """class representing a Name node"""
+
+
+
+
+#####################   node classes   ########################################
+
+class Arguments(NodeNG, AssignTypeMixin):
+    """class representing an Arguments node"""
+    _astng_fields = ('args', 'defaults')
+    args = None
+    defaults = None
+
+    def __init__(self, vararg=None, kwarg=None):
+        self.vararg = vararg
+        self.kwarg = kwarg
+
+    def _infer_name(self, frame, name):
+        if self.parent is frame:
+            return name
+        return None
+
+    def format_args(self):
+        """return arguments formatted as string"""
+        result = [_format_args(self.args, self.defaults)]
+        if self.vararg:
+            result.append('*%s' % self.vararg)
+        if self.kwarg:
+            result.append('**%s' % self.kwarg)
+        return ', '.join(result)
+
+    def default_value(self, argname):
+        """return the default value for an argument
+
+        :raise `NoDefault`: if there is no default value defined
+        """
+        i = _find_arg(argname, self.args)[0]
+        if i is not None:
+            idx = i - (len(self.args) - len(self.defaults))
+            if idx >= 0:
+                return self.defaults[idx]
+        raise NoDefault()
+
+    def is_argument(self, name):
+        """return True if the name is defined in arguments"""
+        if name == self.vararg:
+            return True
+        if name == self.kwarg:
+            return True
+        return self.find_argname(name, True)[1] is not None
+
+    def find_argname(self, argname, rec=False):
+        """return index and Name node with given name"""
+        if self.args: # self.args may be None in some cases (builtin function)
+            return _find_arg(argname, self.args, rec)
+        return None, None
+
+
+def _find_arg(argname, args, rec=False):
+    for i, arg in enumerate(args):
+        if isinstance(arg, Tuple):
+            if rec:
+                found = _find_arg(argname, arg.elts)
+                if found[0] is not None:
+                    return found
+        elif arg.name == argname:
+            return i, arg
+    return None, None
+
+
+def _format_args(args, defaults=None):
+    values = []
+    if args is None:
+        return ''
+    if defaults is not None:
+        default_offset = len(args) - len(defaults)
+    for i, arg in enumerate(args):
+        if isinstance(arg, Tuple):
+            values.append('(%s)' % _format_args(arg.elts))
+        else:
+            values.append(arg.name)
+            if defaults is not None and i >= default_offset:
+                values[-1] += '=' + defaults[i-default_offset].as_string()
+    return ', '.join(values)
+
+
+class AssAttr(NodeNG, ParentAssignTypeMixin):
+    """class representing an AssAttr node"""
+    _astng_fields = ('expr',)
+    expr = None
+
+class Assert(Statement):
+    """class representing an Assert node"""
+    _astng_fields = ('test', 'fail',)
+    test = None
+    fail = None
+
+class Assign(Statement, AssignTypeMixin):
+    """class representing an Assign node"""
+    _astng_fields = ('targets', 'value',)
+    targets = None
+    value = None
+
+class AugAssign(Statement, AssignTypeMixin):
+    """class representing an AugAssign node"""
+    _astng_fields = ('target', 'value',)
+    target = None
+    value = None
+
+class Backquote(NodeNG):
+    """class representing a Backquote node"""
+    _astng_fields = ('value',)
+    value = None
+
+class BinOp(NodeNG):
+    """class representing a BinOp node"""
+    _astng_fields = ('left', 'right',)
+    left = None
+    right = None
+
+class BoolOp(NodeNG):
+    """class representing a BoolOp node"""
+    _astng_fields = ('values',)
+    values = None
+
+class Break(Statement):
+    """class representing a Break node"""
+
+
+class CallFunc(NodeNG):
+    """class representing a CallFunc node"""
+    _astng_fields = ('func', 'args', 'starargs', 'kwargs')
+    func = None
+    args = None
+    starargs = None
+    kwargs = None
+
+    def __init__(self):
+        self.starargs = None
+        self.kwargs = None
+
+class Compare(NodeNG):
+    """class representing a Compare node"""
+    _astng_fields = ('left', 'ops',)
+    left = None
+    ops = None
+
+    def get_children(self):
+        """override get_children for tuple fields"""
+        yield self.left
+        for _, comparator in self.ops:
+            yield comparator # we don't want the 'op'
+
+    def last_child(self):
+        """override last_child"""
+        # XXX maybe if self.ops:
+        return self.ops[-1][1]
+        #return self.left
+
+class Comprehension(NodeNG):
+    """class representing a Comprehension node"""
+    _astng_fields = ('target', 'iter' ,'ifs')
+    target = None
+    iter = None
+    ifs = None
+
+    optional_assign = True
+    def ass_type(self):
+        return self
+
+    def _get_filtered_stmts(self, lookup_node, node, stmts, mystmt):
+        """method used in filter_stmts"""
+        if self is mystmt:
+            if isinstance(lookup_node, (Const, Name)):
+                return [lookup_node], True
+
+        elif self.statement() is mystmt:
+            # original node's statement is the assignment, only keeps
+            # current node (gen exp, list comp)
+
+            return [node], True
+
+        return stmts, False
+
+
+class Const(NodeNG, Instance):
+    """represent a constant node like num, str, bool, None, bytes"""
+
+    def __init__(self, value=None):
+        self.value = value
+
+    def getitem(self, index, context=None):
+        if isinstance(self.value, basestring):
+            return Const(self.value[index])
+        raise TypeError('%r (value=%s)' % (self, self.value))
+
+    def has_dynamic_getattr(self):
+        return False
+
+    def itered(self):
+        if isinstance(self.value, basestring):
+            return self.value
+        raise TypeError()
+
+    def pytype(self):
+        return self._proxied.qname()
+
+
+class Continue(Statement):
+    """class representing a Continue node"""
+
+
+class Decorators(NodeNG):
+    """class representing a Decorators node"""
+    _astng_fields = ('nodes',)
+    nodes = None
+
+    def __init__(self, nodes=None):
+        self.nodes = nodes
+
+    def scope(self):
+        # skip the function node to go directly to the upper level scope
+        return self.parent.parent.scope()
+
+class DelAttr(NodeNG, ParentAssignTypeMixin):
+    """class representing a DelAttr node"""
+    _astng_fields = ('expr',)
+    expr = None
+
+
+class Delete(Statement, AssignTypeMixin):
+    """class representing a Delete node"""
+    _astng_fields = ('targets',)
+    targets = None
+
+
+class Dict(NodeNG, Instance):
+    """class representing a Dict node"""
+    _astng_fields = ('items',)
+
+    def __init__(self, items=None):
+        if items is None:
+            self.items = []
+        else:
+            self.items = [(const_factory(k), const_factory(v))
+                          for k,v in items.iteritems()]
+
+    def pytype(self):
+        return '%s.dict' % BUILTINS_MODULE
+
+    def get_children(self):
+        """get children of a Dict node"""
+        # overrides get_children
+        for key, value in self.items:
+            yield key
+            yield value
+
+    def last_child(self):
+        """override last_child"""
+        if self.items:
+            return self.items[-1][1]
+        return None
+
+    def itered(self):
+        return self.items[::2]
+
+    def getitem(self, key, context=None):
+        for i in xrange(0, len(self.items), 2):
+            for inferedkey in self.items[i].infer(context):
+                if inferedkey is YES:
+                    continue
+                if isinstance(inferedkey, Const) and inferedkey.value == key:
+                    return self.items[i+1]
+        raise IndexError(key)
+
+
+class Discard(Statement):
+    """class representing a Discard node"""
+    _astng_fields = ('value',)
+    value = None
+
+
+class Ellipsis(NodeNG):
+    """class representing an Ellipsis node"""
+
+
+class EmptyNode(NodeNG):
+    """class representing an EmptyNode node"""
+
+
+class ExceptHandler(Statement, AssignTypeMixin):
+    """class representing an ExceptHandler node"""
+    _astng_fields = ('type', 'name', 'body',)
+    type = None
+    name = None
+    body = None
+
+    def _blockstart_toline(self):
+        if self.name:
+            return self.name.tolineno
+        elif self.type:
+            return self.type.tolineno
+        else:
+            return self.lineno
+
+    def set_line_info(self, lastchild):
+        self.fromlineno = self.lineno
+        self.tolineno = lastchild.tolineno
+        self.blockstart_tolineno = self._blockstart_toline()
+
+    def catch(self, exceptions):
+        if self.type is None or exceptions is None:
+            return True
+        for node in self.type.nodes_of_class(Name):
+            if node.name in exceptions:
+                return True
+
+
+class Exec(Statement):
+    """class representing an Exec node"""
+    _astng_fields = ('expr', 'globals', 'locals',)
+    expr = None
+    globals = None
+    locals = None
+
+
+class ExtSlice(NodeNG):
+    """class representing an ExtSlice node"""
+    _astng_fields = ('dims',)
+    dims = None
+
+class For(BlockRangeMixIn, AssignTypeMixin, Statement):
+    """class representing a For node"""
+    _astng_fields = ('target', 'iter', 'body', 'orelse',)
+    target = None
+    iter = None
+    body = None
+    orelse = None
+
+    optional_assign = True
+    def _blockstart_toline(self):
+        return self.iter.tolineno
+
+
+class From(FromImportMixIn, Statement):
+    """class representing a From node"""
+
+    def __init__(self,  fromname, names, level=0):
+        self.modname = fromname
+        self.names = names
+        self.level = level
+
+class Getattr(NodeNG):
+    """class representing a Getattr node"""
+    _astng_fields = ('expr',)
+    expr = None
+
+
+class Global(Statement):
+    """class representing a Global node"""
+
+    def __init__(self, names):
+        self.names = names
+
+    def _infer_name(self, frame, name):
+        return name
+
+
+class If(BlockRangeMixIn, Statement):
+    """class representing an If node"""
+    _astng_fields = ('test', 'body', 'orelse')
+    test = None
+    body = None
+    orelse = None
+
+    def _blockstart_toline(self):
+        return self.test.tolineno
+
+    def block_range(self, lineno):
+        """handle block line numbers range for if statements"""
+        if lineno == self.body[0].fromlineno:
+            return lineno, lineno
+        if lineno <= self.body[-1].tolineno:
+            return lineno, self.body[-1].tolineno
+        return self._elsed_block_range(lineno, self.orelse,
+                                       self.body[0].fromlineno - 1)
+
+
+class IfExp(NodeNG):
+    """class representing an IfExp node"""
+    _astng_fields = ('test', 'body', 'orelse')
+    test = None
+    body = None
+    orelse = None
+
+
+class Import(FromImportMixIn, Statement):
+    """class representing an Import node"""
+
+
+class Index(NodeNG):
+    """class representing an Index node"""
+    _astng_fields = ('value',)
+    value = None
+
+
+class Keyword(NodeNG):
+    """class representing a Keyword node"""
+    _astng_fields = ('value',)
+    value = None
+
+
+class List(NodeNG, Instance, ParentAssignTypeMixin):
+    """class representing a List node"""
+    _astng_fields = ('elts',)
+
+    def __init__(self, elts=None):
+        if elts is None:
+            self.elts = []
+        else:
+            self.elts = [const_factory(e) for e in elts]
+
+    def pytype(self):
+        return '%s.list' % BUILTINS_MODULE
+
+    def getitem(self, index, context=None):
+        return self.elts[index]
+
+    def itered(self):
+        return self.elts
+
+
+class Nonlocal(Statement):
+    """class representing a Nonlocal node"""
+
+    def __init__(self, names):
+        self.names = names
+
+    def _infer_name(self, frame, name):
+        return name
+
+
+class Pass(Statement):
+    """class representing a Pass node"""
+
+
+class Print(Statement):
+    """class representing a Print node"""
+    _astng_fields = ('dest', 'values',)
+    dest = None
+    values = None
+
+
+class Raise(Statement):
+    """class representing a Raise node"""
+    exc = None
+    if sys.version_info < (3, 0):
+        _astng_fields = ('exc', 'inst', 'tback')
+        inst = None
+        tback = None
+    else:
+        _astng_fields = ('exc', 'cause')
+        exc = None
+        cause = None
+
+    def raises_not_implemented(self):
+        if not self.exc:
+            return
+        for name in self.exc.nodes_of_class(Name):
+            if name.name == 'NotImplementedError':
+                return True
+
+
+class Return(Statement):
+    """class representing a Return node"""
+    _astng_fields = ('value',)
+    value = None
+
+
+class Set(NodeNG, Instance, ParentAssignTypeMixin):
+    """class representing a Set node"""
+    _astng_fields = ('elts',)
+
+    def __init__(self, elts=None):
+        if elts is None:
+            self.elts = []
+        else:
+            self.elts = [const_factory(e) for e in elts]
+
+    def pytype(self):
+        return '%s.set' % BUILTINS_MODULE
+
+    def itered(self):
+        return self.elts
+
+
+class Slice(NodeNG):
+    """class representing a Slice node"""
+    _astng_fields = ('lower', 'upper', 'step')
+    lower = None
+    upper = None
+    step = None
+
+class Starred(NodeNG):
+    """class representing a Starred node"""
+    _astng_fields = ('value',)
+    value = None
+
+
+class Subscript(NodeNG):
+    """class representing a Subscript node"""
+    _astng_fields = ('value', 'slice')
+    value = None
+    slice = None
+
+
+class TryExcept(BlockRangeMixIn, Statement):
+    """class representing a TryExcept node"""
+    _astng_fields = ('body', 'handlers', 'orelse',)
+    body = None
+    handlers = None
+    orelse = None
+
+    def _infer_name(self, frame, name):
+        return name
+
+    def _blockstart_toline(self):
+        return self.lineno
+
+    def block_range(self, lineno):
+        """handle block line numbers range for try/except statements"""
+        last = None
+        for exhandler in self.handlers:
+            if exhandler.type and lineno == exhandler.type.fromlineno:
+                return lineno, lineno
+            if exhandler.body[0].fromlineno <= lineno <= exhandler.body[-1].tolineno:
+                return lineno, exhandler.body[-1].tolineno
+            if last is None:
+                last = exhandler.body[0].fromlineno - 1
+        return self._elsed_block_range(lineno, self.orelse, last)
+
+
+class TryFinally(BlockRangeMixIn, Statement):
+    """class representing a TryFinally node"""
+    _astng_fields = ('body', 'finalbody',)
+    body = None
+    finalbody = None
+
+    def _blockstart_toline(self):
+        return self.lineno
+
+    def block_range(self, lineno):
+        """handle block line numbers range for try/finally statements"""
+        child = self.body[0]
+        # py2.5 try: except: finally:
+        if (isinstance(child, TryExcept) and child.fromlineno == self.fromlineno
+            and lineno > self.fromlineno and lineno <= child.tolineno):
+            return child.block_range(lineno)
+        return self._elsed_block_range(lineno, self.finalbody)
+
+
+class Tuple(NodeNG, Instance, ParentAssignTypeMixin):
+    """class representing a Tuple node"""
+    _astng_fields = ('elts',)
+
+    def __init__(self, elts=None):
+        if elts is None:
+            self.elts = []
+        else:
+            self.elts = [const_factory(e) for e in elts]
+
+    def pytype(self):
+        return '%s.tuple' % BUILTINS_MODULE
+
+    def getitem(self, index, context=None):
+        return self.elts[index]
+
+    def itered(self):
+        return self.elts
+
+
+class UnaryOp(NodeNG):
+    """class representing an UnaryOp node"""
+    _astng_fields = ('operand',)
+    operand = None
+
+
+class While(BlockRangeMixIn, Statement):
+    """class representing a While node"""
+    _astng_fields = ('test', 'body', 'orelse',)
+    test = None
+    body = None
+    orelse = None
+
+    def _blockstart_toline(self):
+        return self.test.tolineno
+
+    def block_range(self, lineno):
+        """handle block line numbers range for for and while statements"""
+        return self. _elsed_block_range(lineno, self.orelse)
+
+
+class With(BlockRangeMixIn, AssignTypeMixin, Statement):
+    """class representing a With node"""
+    _astng_fields = ('expr', 'vars', 'body')
+    expr = None
+    vars = None
+    body = None
+
+    def _blockstart_toline(self):
+        if self.vars:
+            return self.vars.tolineno
+        else:
+            return self.expr.tolineno
+
+
+class Yield(NodeNG):
+    """class representing a Yield node"""
+    _astng_fields = ('value',)
+    value = None
+
+# constants ##############################################################
+
+CONST_CLS = {
+    list: List,
+    tuple: Tuple,
+    dict: Dict,
+    set: Set,
+    type(None): Const,
+    }
+
+def _update_const_classes():
+    """update constant classes, so the keys of CONST_CLS can be reused"""
+    klasses = (bool, int, float, complex, str)
+    if sys.version_info < (3, 0):
+        klasses += (unicode, long)
+    if sys.version_info >= (2, 6):
+        klasses += (bytes,)
+    for kls in klasses:
+        CONST_CLS[kls] = Const
+_update_const_classes()
+
+def const_factory(value):
+    """return an astng node for a python value"""
+    # since const_factory is called to evaluate content of container (eg list,
+    # tuple), it may be called with some node as argument that should be left
+    # untouched
+    if isinstance(value, NodeNG):
+        return value
+    try:
+        return CONST_CLS[value.__class__](value)
+    except (KeyError, AttributeError):
+        # some constants (like from gtk._gtk) don't have their class in
+        # CONST_CLS, though we can "assert isinstance(value, tuple(CONST_CLS))"
+        if isinstance(value, tuple(CONST_CLS)):
+            return Const(value)
+        node = EmptyNode()
+        node.object = value
+        return node

pylibs/logilab/astng/nodes.py

@@ -0,0 +1,75 @@
+# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+# copyright 2003-2010 Sylvain Thenault, all rights reserved.
+# contact mailto:thenault@gmail.com
+#
+# This file is part of logilab-astng.
+#
+# logilab-astng is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Lesser General Public License as published by the
+# Free Software Foundation, either version 2.1 of the License, or (at your
+# option) any later version.
+#
+# logilab-astng is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License
+# for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with logilab-astng. If not, see <http://www.gnu.org/licenses/>.
+"""
+on all nodes :
+ .is_statement, returning true if the node should be considered as a
+  statement node
+ .root(), returning the root node of the tree (i.e. a Module)
+ .previous_sibling(), returning previous sibling statement node
+ .next_sibling(), returning next sibling statement node
+ .statement(), returning the first parent node marked as statement node
+ .frame(), returning the first node defining a new local scope (i.e.
+  Module, Function or Class)
+ .set_local(name, node), define an identifier <name> on the first parent frame,
+  with the node defining it. This is used by the astng builder and should not
+  be used from out there.
+
+on From and Import :
+ .real_name(name),
+
+
+"""
+
+__docformat__ = "restructuredtext en"
+
+from logilab.astng.node_classes import Arguments, AssAttr, Assert, Assign, \
+    AssName, AugAssign, Backquote, BinOp, BoolOp, Break, CallFunc, Compare, \
+    Comprehension, Const, Continue, Decorators, DelAttr, DelName, Delete, \
+    Dict, Discard, Ellipsis, EmptyNode, ExceptHandler, Exec, ExtSlice, For, \
+    From, Getattr, Global, If, IfExp, Import, Index, Keyword, \
+    List, Name, Nonlocal, Pass, Print, Raise, Return, Set, Slice, Starred, Subscript, \
+    TryExcept, TryFinally, Tuple, UnaryOp, While, With, Yield, \
+    const_factory
+from logilab.astng.scoped_nodes import Module, GenExpr, Lambda, DictComp, \
+    ListComp, SetComp, Function, Class
+
+ALL_NODE_CLASSES = (
+    Arguments, AssAttr, Assert, Assign, AssName, AugAssign,
+    Backquote, BinOp, BoolOp, Break,
+    CallFunc, Class, Compare, Comprehension, Const, Continue,
+    Decorators, DelAttr, DelName, Delete,
+    Dict, DictComp, Discard,
+    Ellipsis, EmptyNode, ExceptHandler, Exec, ExtSlice,
+    For, From, Function,
+    Getattr, GenExpr, Global,
+    If, IfExp, Import, Index,
+    Keyword,
+    Lambda, List, ListComp,
+    Name, Nonlocal,
+    Module,
+    Pass, Print,
+    Raise, Return,
+    Set, SetComp, Slice, Starred, Subscript,
+    TryExcept, TryFinally, Tuple,
+    UnaryOp,
+    While, With,
+    Yield,
+    )
+

pylibs/logilab/astng/protocols.py

@@ -0,0 +1,321 @@
+# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+# copyright 2003-2010 Sylvain Thenault, all rights reserved.
+# contact mailto:thenault@gmail.com
+#
+# This file is part of logilab-astng.
+#
+# logilab-astng is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Lesser General Public License as published by the
+# Free Software Foundation, either version 2.1 of the License, or (at your
+# option) any later version.
+#
+# logilab-astng is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License
+# for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with logilab-astng. If not, see <http://www.gnu.org/licenses/>.
+"""this module contains a set of functions to handle python protocols for nodes
+where it makes sense.
+"""
+
+__doctype__ = "restructuredtext en"
+
+from logilab.astng.exceptions import InferenceError, NoDefault
+from logilab.astng.node_classes import unpack_infer
+from logilab.astng.bases import copy_context, \
+     raise_if_nothing_infered, yes_if_nothing_infered, Instance, Generator, YES
+from logilab.astng.nodes import const_factory
+from logilab.astng import nodes
+
+# unary operations ############################################################
+
+def tl_infer_unary_op(self, operator):
+    if operator == 'not':
+        return const_factory(not bool(self.elts))
+    raise TypeError() # XXX log unsupported operation
+nodes.Tuple.infer_unary_op = tl_infer_unary_op
+nodes.List.infer_unary_op = tl_infer_unary_op
+
+
+def dict_infer_unary_op(self, operator):
+    if operator == 'not':
+        return const_factory(not bool(self.items))
+    raise TypeError() # XXX log unsupported operation
+nodes.Dict.infer_unary_op = dict_infer_unary_op
+
+
+def const_infer_unary_op(self, operator):
+    if operator == 'not':
+        return const_factory(not self.value)
+    # XXX log potentially raised TypeError
+    elif operator == '+':
+        return const_factory(+self.value)
+    else: # operator == '-':
+        return const_factory(-self.value)
+nodes.Const.infer_unary_op = const_infer_unary_op
+
+
+# binary operations ###########################################################
+
+BIN_OP_IMPL = {'+':  lambda a, b: a + b,
+               '-':  lambda a, b: a - b,
+               '/':  lambda a, b: a / b,
+               '//': lambda a, b: a // b,
+               '*':  lambda a, b: a * b,
+               '**': lambda a, b: a ** b,
+               '%':  lambda a, b: a % b,
+               '&':  lambda a, b: a & b,
+               '|':  lambda a, b: a | b,
+               '^':  lambda a, b: a ^ b,
+               '<<': lambda a, b: a << b,
+               '>>': lambda a, b: a >> b,
+               }
+for key, impl in BIN_OP_IMPL.items():
+    BIN_OP_IMPL[key+'='] = impl
+
+def const_infer_binary_op(self, operator, other, context):
+    for other in other.infer(context):
+        if isinstance(other, nodes.Const):
+            try:
+                impl = BIN_OP_IMPL[operator]
+
+                try:
+                    yield const_factory(impl(self.value, other.value))
+                except Exception:
+                    # ArithmeticError is not enough: float >> float is a TypeError
+                    # TODO : let pylint know about the problem
+                    pass
+            except TypeError:
+                # XXX log TypeError
+                continue
+        elif other is YES:
+            yield other
+        else:
+            try:
+                for val in other.infer_binary_op(operator, self, context):
+                    yield val
+            except AttributeError:
+                yield YES
+nodes.Const.infer_binary_op = yes_if_nothing_infered(const_infer_binary_op)
+
+
+def tl_infer_binary_op(self, operator, other, context):
+    for other in other.infer(context):
+        if isinstance(other, self.__class__) and operator == '+':
+            node = self.__class__()
+            elts = [n for elt in self.elts for n in elt.infer(context)
+                    if not n is YES]
+            elts += [n for elt in other.elts for n in elt.infer(context)
+                     if not n is YES]
+            node.elts = elts
+            yield node
+        elif isinstance(other, nodes.Const) and operator == '*':
+            if not isinstance(other.value, int):
+                yield YES
+                continue
+            node = self.__class__()
+            elts = [n for elt in self.elts for n in elt.infer(context)
+                    if not n is YES] * other.value
+            node.elts = elts
+            yield node
+        elif isinstance(other, Instance) and not isinstance(other, nodes.Const):
+            yield YES
+    # XXX else log TypeError
+nodes.Tuple.infer_binary_op = yes_if_nothing_infered(tl_infer_binary_op)
+nodes.List.infer_binary_op = yes_if_nothing_infered(tl_infer_binary_op)
+
+
+def dict_infer_binary_op(self, operator, other, context):
+    for other in other.infer(context):
+        if isinstance(other, Instance) and isinstance(other._proxied, nodes.Class):
+            yield YES
+        # XXX else log TypeError
+nodes.Dict.infer_binary_op = yes_if_nothing_infered(dict_infer_binary_op)
+
+
+# assignment ##################################################################
+
+"""the assigned_stmts method is responsible to return the assigned statement
+(e.g. not inferred) according to the assignment type.
+
+The `asspath` argument is used to record the lhs path of the original node.
+For instance if we want assigned statements for 'c' in 'a, (b,c)', asspath
+will be [1, 1] once arrived to the Assign node.
+
+The `context` argument is the current inference context which should be given
+to any intermediary inference necessary.
+"""
+
+def _resolve_looppart(parts, asspath, context):
+    """recursive function to resolve multiple assignments on loops"""
+    asspath = asspath[:]
+    index = asspath.pop(0)
+    for part in parts:
+        if part is YES:
+            continue
+        # XXX handle __iter__ and log potentially detected errors
+        if not hasattr(part, 'itered'):
+            continue
+        try:
+            itered = part.itered()
+        except TypeError:
+            continue # XXX log error
+        for stmt in itered:
+            try:
+                assigned = stmt.getitem(index, context)
+            except (AttributeError, IndexError):
+                continue
+            except TypeError, exc: # stmt is unsubscriptable Const
+                continue
+            if not asspath:
+                # we achieved to resolved the assignment path,
+                # don't infer the last part
+                yield assigned
+            elif assigned is YES:
+                break
+            else:
+                # we are not yet on the last part of the path
+                # search on each possibly inferred value
+                try:
+                    for infered in _resolve_looppart(assigned.infer(context),
+                                                     asspath, context):
+                        yield infered
+                except InferenceError:
+                    break
+
+
+def for_assigned_stmts(self, node, context=None, asspath=None):
+    if asspath is None:
+        for lst in self.iter.infer(context):
+            if isinstance(lst, (nodes.Tuple, nodes.List)):
+                for item in lst.elts:
+                    yield item
+    else:
+        for infered in _resolve_looppart(self.iter.infer(context),
+                                         asspath, context):
+            yield infered
+
+nodes.For.assigned_stmts = raise_if_nothing_infered(for_assigned_stmts)
+nodes.Comprehension.assigned_stmts = raise_if_nothing_infered(for_assigned_stmts)
+
+
+def mulass_assigned_stmts(self, node, context=None, asspath=None):
+    if asspath is None:
+        asspath = []
+    asspath.insert(0, self.elts.index(node))
+    return self.parent.assigned_stmts(self, context, asspath)
+nodes.Tuple.assigned_stmts = mulass_assigned_stmts
+nodes.List.assigned_stmts = mulass_assigned_stmts
+
+
+def assend_assigned_stmts(self, context=None):
+    return self.parent.assigned_stmts(self, context=context)
+nodes.AssName.assigned_stmts = assend_assigned_stmts
+nodes.AssAttr.assigned_stmts = assend_assigned_stmts
+
+
+def _arguments_infer_argname(self, name, context):
+    # arguments information may be missing, in which case we can't do anything
+    # more
+    if not (self.args or self.vararg or self.kwarg):
+        yield YES
+        return
+    # first argument of instance/class method
+    if self.args and getattr(self.args[0], 'name', None) == name:
+        functype = self.parent.type
+        if functype == 'method':
+            yield Instance(self.parent.parent.frame())
+            return
+        if functype == 'classmethod':
+            yield self.parent.parent.frame()
+            return
+    if name == self.vararg:
+        yield const_factory(())
+        return
+    if name == self.kwarg:
+        yield const_factory({})
+        return
+    # if there is a default value, yield it. And then yield YES to reflect
+    # we can't guess given argument value
+    try:
+        context = copy_context(context)
+        for infered in self.default_value(name).infer(context):
+            yield infered
+        yield YES
+    except NoDefault:
+        yield YES
+
+
+def arguments_assigned_stmts(self, node, context, asspath=None):
+    if context.callcontext:
+        # reset call context/name
+        callcontext = context.callcontext
+        context = copy_context(context)
+        context.callcontext = None
+        for infered in callcontext.infer_argument(self.parent, node.name, context):
+            yield infered
+        return
+    for infered in _arguments_infer_argname(self, node.name, context):
+        yield infered
+nodes.Arguments.assigned_stmts = arguments_assigned_stmts
+
+
+def assign_assigned_stmts(self, node, context=None, asspath=None):
+    if not asspath:
+        yield self.value
+        return
+    for infered in _resolve_asspart(self.value.infer(context), asspath, context):
+        yield infered
+nodes.Assign.assigned_stmts = raise_if_nothing_infered(assign_assigned_stmts)
+nodes.AugAssign.assigned_stmts = raise_if_nothing_infered(assign_assigned_stmts)
+
+
+def _resolve_asspart(parts, asspath, context):
+    """recursive function to resolve multiple assignments"""
+    asspath = asspath[:]
+    index = asspath.pop(0)
+    for part in parts:
+        if hasattr(part, 'getitem'):
+            try:
+                assigned = part.getitem(index, context)
+            # XXX raise a specific exception to avoid potential hiding of
+            # unexpected exception ?
+            except (TypeError, IndexError):
+                return
+            if not asspath:
+                # we achieved to resolved the assignment path, don't infer the
+                # last part
+                yield assigned
+            elif assigned is YES:
+                return
+            else:
+                # we are not yet on the last part of the path search on each
+                # possibly inferred value
+                try:
+                    for infered in _resolve_asspart(assigned.infer(context),
+                                                    asspath, context):
+                        yield infered
+                except InferenceError:
+                    return
+
+
+def excepthandler_assigned_stmts(self, node, context=None, asspath=None):
+    for assigned in unpack_infer(self.type):
+        if isinstance(assigned, nodes.Class):
+            assigned = Instance(assigned)
+        yield assigned
+nodes.ExceptHandler.assigned_stmts = raise_if_nothing_infered(excepthandler_assigned_stmts)
+
+
+def with_assigned_stmts(self, node, context=None, asspath=None):
+    if asspath is None:
+        for lst in self.vars.infer(context):
+            if isinstance(lst, (nodes.Tuple, nodes.List)):
+                for item in lst.nodes:
+                    yield item
+nodes.With.assigned_stmts = raise_if_nothing_infered(with_assigned_stmts)
+
+

pylibs/logilab/astng/raw_building.py

@@ -0,0 +1,349 @@
+# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+# copyright 2003-2010 Sylvain Thenault, all rights reserved.
+# contact mailto:thenault@gmail.com
+#
+# This file is part of logilab-astng.
+#
+# logilab-astng is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Lesser General Public License as published by the
+# Free Software Foundation, either version 2.1 of the License, or (at your
+# option) any later version.
+#
+# logilab-astng is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License
+# for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with logilab-astng. If not, see <http://www.gnu.org/licenses/>.
+"""this module contains a set of functions to create astng trees from scratch
+(build_* functions) or from living object (object_build_* functions)
+"""
+
+__docformat__ = "restructuredtext en"
+
+import sys
+from os.path import abspath
+from inspect import (getargspec, isdatadescriptor, isfunction, ismethod,
+                     ismethoddescriptor, isclass, isbuiltin)
+
+from logilab.astng import BUILTINS_MODULE
+from logilab.astng.node_classes import CONST_CLS
+from logilab.astng.nodes import (Module, Class, Const, const_factory, From,
+    Function, EmptyNode, Name, Arguments, Dict, List, Set, Tuple)
+from logilab.astng.bases import Generator
+from logilab.astng.manager import ASTNGManager
+MANAGER = ASTNGManager()
+
+_CONSTANTS = tuple(CONST_CLS) # the keys of CONST_CLS eg python builtin types
+
+def _attach_local_node(parent, node, name):
+    node.name = name # needed by add_local_node
+    parent.add_local_node(node)
+
+_marker = object()
+
+def attach_dummy_node(node, name, object=_marker):
+    """create a dummy node and register it in the locals of the given
+    node with the specified name
+    """
+    enode = EmptyNode()
+    enode.object = object
+    _attach_local_node(node, enode, name)
+
+EmptyNode.has_underlying_object = lambda self: self.object is not _marker
+
+def attach_const_node(node, name, value):
+    """create a Const node and register it in the locals of the given
+    node with the specified name
+    """
+    if not name in node.special_attributes:
+        _attach_local_node(node, const_factory(value), name)
+
+def attach_import_node(node, modname, membername):
+    """create a From node and register it in the locals of the given
+    node with the specified name
+    """
+    from_node = From(modname, [(membername, None)])
+    _attach_local_node(node, from_node, membername)
+
+
+def build_module(name, doc=None):
+    """create and initialize a astng Module node"""
+    node = Module(name, doc, pure_python=False)
+    node.package = False
+    node.parent = None
+    return node
+
+def build_class(name, basenames=(), doc=None):
+    """create and initialize a astng Class node"""
+    node = Class(name, doc)
+    for base in basenames:
+        basenode = Name()
+        basenode.name = base
+        node.bases.append(basenode)
+        basenode.parent = node
+    return node
+
+def build_function(name, args=None, defaults=None, flag=0, doc=None):
+    """create and initialize a astng Function node"""
+    args, defaults = args or [], defaults or []
+    # first argument is now a list of decorators
+    func = Function(name, doc)
+    func.args = argsnode = Arguments()
+    argsnode.args = []
+    for arg in args:
+        argsnode.args.append(Name())
+        argsnode.args[-1].name = arg
+        argsnode.args[-1].parent = argsnode
+    argsnode.defaults = []
+    for default in defaults:
+        argsnode.defaults.append(const_factory(default))
+        argsnode.defaults[-1].parent = argsnode
+    argsnode.kwarg = None
+    argsnode.vararg = None
+    argsnode.parent = func
+    if args:
+        register_arguments(func)
+    return func
+
+
+def build_from_import(fromname, names):
+    """create and initialize an astng From import statement"""
+    return From(fromname, [(name, None) for name in names])
+
+def register_arguments(func, args=None):
+    """add given arguments to local
+
+    args is a list that may contains nested lists
+    (i.e. def func(a, (b, c, d)): ...)
+    """
+    if args is None:
+        args = func.args.args
+        if func.args.vararg:
+            func.set_local(func.args.vararg, func.args)
+        if func.args.kwarg:
+            func.set_local(func.args.kwarg, func.args)
+    for arg in args:
+        if isinstance(arg, Name):
+            func.set_local(arg.name, arg)
+        else:
+            register_arguments(func, arg.elts)
+
+def object_build_class(node, member, localname):
+    """create astng for a living class object"""
+    basenames = [base.__name__ for base in member.__bases__]
+    return _base_class_object_build(node, member, basenames,
+                                    localname=localname)
+
+def object_build_function(node, member, localname):
+    """create astng for a living function object"""
+    args, varargs, varkw, defaults = getargspec(member)
+    if varargs is not None:
+        args.append(varargs)
+    if varkw is not None:
+        args.append(varkw)
+    func = build_function(getattr(member, '__name__', None) or localname, args,
+                          defaults, member.func_code.co_flags, member.__doc__)
+    node.add_local_node(func, localname)
+
+def object_build_datadescriptor(node, member, name):
+    """create astng for a living data descriptor object"""
+    return _base_class_object_build(node, member, [], name)
+
+def object_build_methoddescriptor(node, member, localname):
+    """create astng for a living method descriptor object"""
+    # FIXME get arguments ?
+    func = build_function(getattr(member, '__name__', None) or localname,
+                          doc=member.__doc__)
+    # set node's arguments to None to notice that we have no information, not
+    # and empty argument list
+    func.args.args = None
+    node.add_local_node(func, localname)
+
+def _base_class_object_build(node, member, basenames, name=None, localname=None):
+    """create astng for a living class object, with a given set of base names
+    (e.g. ancestors)
+    """
+    klass = build_class(name or getattr(member, '__name__', None) or localname,
+                        basenames, member.__doc__)
+    klass._newstyle = isinstance(member, type)
+    node.add_local_node(klass, localname)
+    try:
+        # limit the instantiation trick since it's too dangerous
+        # (such as infinite test execution...)
+        # this at least resolves common case such as Exception.args,
+        # OSError.errno
+        if issubclass(member, Exception):
+            instdict = member().__dict__
+        else:
+            raise TypeError
+    except:
+        pass
+    else:
+        for name, obj in instdict.items():
+            valnode = EmptyNode()
+            valnode.object = obj
+            valnode.parent = klass
+            valnode.lineno = 1
+            klass.instance_attrs[name] = [valnode]
+    return klass
+
+
+
+
+class InspectBuilder(object):
+    """class for building nodes from living object
+
+    this is actually a really minimal representation, including only Module,
+    Function and Class nodes and some others as guessed.
+    """
+
+    # astng from living objects ###############################################
+
+    def __init__(self):
+        self._done = {}
+        self._module = None
+
+    def inspect_build(self, module, modname=None, path=None):
+        """build astng from a living module (i.e. using inspect)
+        this is used when there is no python source code available (either
+        because it's a built-in module or because the .py is not available)
+        """
+        self._module = module
+        if modname is None:
+            modname = module.__name__
+        node = build_module(modname, module.__doc__)
+        node.file = node.path = path and abspath(path) or path
+        MANAGER.astng_cache[modname] = node
+        node.package = hasattr(module, '__path__')
+        self._done = {}
+        self.object_build(node, module)
+        return node
+
+    def object_build(self, node, obj):
+        """recursive method which create a partial ast from real objects
+         (only function, class, and method are handled)
+        """
+        if obj in self._done:
+            return self._done[obj]
+        self._done[obj] = node
+        for name in dir(obj):
+            try:
+                member = getattr(obj, name)
+            except AttributeError:
+                # damned ExtensionClass.Base, I know you're there !
+                attach_dummy_node(node, name)
+                continue
+            if ismethod(member):
+                member = member.im_func
+            if isfunction(member):
+                # verify this is not an imported function
+                filename = getattr(member.func_code, 'co_filename', None)
+                if filename is None:
+                    assert isinstance(member, object)
+                    object_build_methoddescriptor(node, member, name)
+                elif filename != getattr(self._module, '__file__', None):
+                    attach_dummy_node(node, name, member)
+                else:
+                    object_build_function(node, member, name)
+            elif isbuiltin(member):
+                if self.imported_member(node, member, name):
+                    #if obj is object:
+                    #    print 'skippp', obj, name, member
+                    continue
+                object_build_methoddescriptor(node, member, name)
+            elif isclass(member):
+                if self.imported_member(node, member, name):
+                    continue
+                if member in self._done:
+                    class_node = self._done[member]
+                    if not class_node in node.locals.get(name, ()):
+                        node.add_local_node(class_node, name)
+                else:
+                    class_node = object_build_class(node, member, name)
+                    # recursion
+                    self.object_build(class_node, member)
+                if name == '__class__' and class_node.parent is None:
+                    class_node.parent = self._done[self._module]
+            elif ismethoddescriptor(member):
+                assert isinstance(member, object)
+                object_build_methoddescriptor(node, member, name)
+            elif isdatadescriptor(member):
+                assert isinstance(member, object)
+                object_build_datadescriptor(node, member, name)
+            elif isinstance(member, _CONSTANTS):
+                attach_const_node(node, name, member)
+            else:
+                # create an empty node so that the name is actually defined
+                attach_dummy_node(node, name, member)
+
+    def imported_member(self, node, member, name):
+        """verify this is not an imported class or handle it"""
+        # /!\ some classes like ExtensionClass doesn't have a __module__
+        # attribute ! Also, this may trigger an exception on badly built module
+        # (see http://www.logilab.org/ticket/57299 for instance)
+        try:
+            modname = getattr(member, '__module__', None)
+        except:
+            # XXX use logging
+            print 'unexpected error while building astng from living object'
+            import traceback
+            traceback.print_exc()
+            modname = None
+        if modname is None:
+            if name in ('__new__', '__subclasshook__'):
+                # Python 2.5.1 (r251:54863, Sep  1 2010, 22:03:14)
+                # >>> print object.__new__.__module__
+                # None
+                modname = BUILTINS_MODULE
+            else:
+                attach_dummy_node(node, name, member)
+                return True
+        if {'gtk': 'gtk._gtk'}.get(modname, modname) != self._module.__name__:
+            # check if it sounds valid and then add an import node, else use a
+            # dummy node
+            try:
+                getattr(sys.modules[modname], name)
+            except (KeyError, AttributeError):
+                attach_dummy_node(node, name, member)
+            else:
+                attach_import_node(node, modname, name)
+            return True
+        return False
+
+
+### astng boot strapping ################################################### ###
+
+_CONST_PROXY = {}
+def astng_boot_strapping():
+    """astng boot strapping the builtins module"""
+    # this boot strapping is necessary since we need the Const nodes to
+    # inspect_build builtins, and then we can proxy Const
+    builder = InspectBuilder()
+    from logilab.common.compat import builtins
+    astng_builtin = builder.inspect_build(builtins)
+    for cls, node_cls in CONST_CLS.items():
+        if cls is type(None):
+            proxy = build_class('NoneType')
+            proxy.parent = astng_builtin
+        else:
+            proxy = astng_builtin.getattr(cls.__name__)[0] # XXX
+        if cls in (dict, list, set, tuple):
+            node_cls._proxied = proxy
+        else:
+            _CONST_PROXY[cls] = proxy
+
+astng_boot_strapping()
+
+# TODO : find a nicer way to handle this situation;
+# However __proxied introduced an
+# infinite recursion (see https://bugs.launchpad.net/pylint/+bug/456870)
+def _set_proxied(const):
+    return _CONST_PROXY[const.value.__class__]
+Const._proxied = property(_set_proxied)
+
+# FIXME : is it alright that Generator._proxied is not a astng node?
+Generator._proxied = MANAGER.infer_astng_from_something(type(a for a in ()))
+

pylibs/logilab/astng/rebuilder.py

@@ -0,0 +1,864 @@
+# copyright 2003-2010 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+# copyright 2003-2010 Sylvain Thenault, all rights reserved.
+# contact mailto:thenault@gmail.com
+#
+# This file is part of logilab-astng.
+#
+# logilab-astng is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Lesser General Public License as published by the
+# Free Software Foundation, either version 2.1 of the License, or (at your
+# option) any later version.
+#
+# logilab-astng is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License
+# for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with logilab-astng. If not, see <http://www.gnu.org/licenses/>.
+"""this module contains utilities for rebuilding a _ast tree in
+order to get a single ASTNG representation
+"""
+
+import sys
+from _ast import (Expr as Discard, Str,
+    # binary operators
+    Add, Div, FloorDiv,  Mod, Mult, Pow, Sub, BitAnd, BitOr, BitXor,
+    LShift, RShift,
+    # logical operators
+    And, Or,
+    # unary operators
+    UAdd, USub, Not, Invert,
+    # comparison operators
+    Eq, Gt, GtE, In, Is, IsNot, Lt, LtE, NotEq, NotIn,
+    )
+
+from logilab.astng.exceptions import ASTNGBuildingException
+from logilab.astng import nodes as new
+
+
+_BIN_OP_CLASSES = {Add: '+',
+                   BitAnd: '&',
+                   BitOr: '|',
+                   BitXor: '^',
+                   Div: '/',
+                   FloorDiv: '//',
+                   Mod: '%',
+                   Mult: '*',
+                   Pow: '**',
+                   Sub: '-',
+                   LShift: '<<',
+                   RShift: '>>'}
+
+_BOOL_OP_CLASSES = {And: 'and',
+                    Or: 'or'}
+
+_UNARY_OP_CLASSES = {UAdd: '+',
+                     USub: '-',
+                     Not: 'not',
+                     Invert: '~'}
+
+_CMP_OP_CLASSES = {Eq: '==',
+                   Gt: '>',
+                   GtE: '>=',
+                   In: 'in',
+                   Is: 'is',
+                   IsNot: 'is not',
+                   Lt: '<',
+                   LtE: '<=',
+                   NotEq: '!=',
+                   NotIn: 'not in'}
+
+CONST_NAME_TRANSFORMS = {'None':  None,
+                         'True':  True,
+                         'False': False}
+
+REDIRECT = {'arguments': 'Arguments',
+            'Attribute': 'Getattr',
+            'comprehension': 'Comprehension',
+            'Call': 'CallFunc',
+            'ClassDef': 'Class',
+            "ListCompFor": 'Comprehension',
+            "GenExprFor": 'Comprehension',
+            'excepthandler': 'ExceptHandler',
+            'Expr': 'Discard',
+            'FunctionDef': 'Function',
+            'GeneratorExp': 'GenExpr',
+            'ImportFrom': 'From',
+            'keyword': 'Keyword',
+            'Repr': 'Backquote',
+            }
+
+def _init_set_doc(node, newnode):
+    newnode.doc = None
+    try:
+        if isinstance(node.body[0], Discard) and isinstance(node.body[0].value, Str):
+            newnode.tolineno = node.body[0].lineno
+            newnode.doc = node.body[0].value.s
+            node.body = node.body[1:]
+
+    except IndexError:
+        pass # ast built from scratch
+
+def _lineno_parent(oldnode, newnode, parent):
+    newnode.parent = parent
+    if hasattr(oldnode, 'lineno'):
+        newnode.lineno = oldnode.lineno
+    if hasattr(oldnode, 'col_offset'):
+        newnode.col_offset = oldnode.col_offset
+
+def _set_infos(oldnode, newnode, parent):
+    newnode.parent = parent
+    if hasattr(oldnode, 'lineno'):
+        newnode.lineno = oldnode.lineno
+    if hasattr(oldnode, 'col_offset'):
+        newnode.col_offset = oldnode.col_offset
+    newnode.set_line_info(newnode.last_child()) # set_line_info accepts None
+
+
+
+
+class TreeRebuilder(object):
+    """Rebuilds the _ast tree to become an ASTNG tree"""
+
+    _visit_meths = {}
+    def __init__(self):
+        self.init()
+
+    def init(self):
+        self.asscontext = None
+        self._metaclass = ['']
+        self._global_names = []
+        self._from_nodes = []
+        self._delayed_assattr = []
+
+    def visit(self, node, parent):
+        cls = node.__class__
+        if cls in self._visit_meths:
+            return self._visit_meths[cls](node, parent)
+        else:
+            cls_name = cls.__name__
+            visit_name = 'visit_' + REDIRECT.get(cls_name, cls_name).lower()
+            visit_method = getattr(self, visit_name)
+            self._visit_meths[cls] = visit_method
+            return visit_method(node, parent)
+
+    def _save_assignment(self, node, name=None):
+        """save assignement situation since node.parent is not available yet"""
+        if self._global_names and node.name in self._global_names[-1]:
+            node.root().set_local(node.name, node)
+        else:
+            node.parent.set_local(node.name, node)
+
+
+    def visit_arguments(self, node, parent):
+        """visit a Arguments node by returning a fresh instance of it"""
+        newnode = new.Arguments()
+        _lineno_parent(node, newnode, parent)
+        self.asscontext = "Ass"
+        newnode.args = [self.visit(child, newnode) for child in node.args]
+        self.asscontext = None
+        newnode.defaults = [self.visit(child, newnode) for child in node.defaults]
+        newnode.vararg = node.vararg
+        newnode.kwarg = node.kwarg
+        # save argument names in locals:
+        if node.vararg:
+            newnode.parent.set_local(newnode.vararg, newnode)
+        if node.kwarg:
+            newnode.parent.set_local(newnode.kwarg, newnode)
+        newnode.set_line_info(newnode.last_child())
+        return newnode
+
+    def visit_assattr(self, node, parent):
+        """visit a AssAttr node by returning a fresh instance of it"""
+        assc, self.asscontext = self.asscontext, None
+        newnode = new.AssAttr()
+        _lineno_parent(node, newnode, parent)
+        newnode.expr = self.visit(node.expr, newnode)
+        self.asscontext = assc
+        self._delayed_assattr.append(newnode)
+        newnode.set_line_info(newnode.last_child())
+        return newnode
+
+    def visit_assert(self, node, parent):
+        """visit a Assert node by returning a fresh instance of it"""
+        newnode = new.Assert()
+        _lineno_parent(node, newnode, parent)
+        newnode.test = self.visit(node.test, newnode)
+        if node.msg is not None:
+            newnode.fail = self.visit(node.msg, newnode)
+        newnode.set_line_info(newnode.last_child())
+        return newnode
+
+    def visit_assign(self, node, parent):
+        """visit a Assign node by returning a fresh instance of it"""
+        newnode = new.Assign()
+        _lineno_parent(node, newnode, parent)
+        self.asscontext = "Ass"
+        newnode.targets = [self.visit(child, newnode) for child in node.targets]
+        self.asscontext = None
+        newnode.value = self.visit(node.value, newnode)
+        # set some function or metaclass infos  XXX explain ?
+        klass = newnode.parent.frame()
+        if (isinstance(klass, new.Class)
+            and isinstance(newnode.value, new.CallFunc)
+            and isinstance(newnode.value.func, new.Name)):
+            func_name = newnode.value.func.name
+            for ass_node in newnode.targets:
+                try:
+                    meth = klass[ass_node.name]
+                    if isinstance(meth, new.Function):
+                        if func_name in ('classmethod', 'staticmethod'):
+                            meth.type = func_name
+                        elif func_name == 'classproperty': # see lgc.decorators
+                            meth.type = 'classmethod'
+                        meth.extra_decorators.append(newnode.value)
+                except (AttributeError, KeyError):
+                    continue
+        elif getattr(newnode.targets[0], 'name', None) == '__metaclass__':
+            # XXX check more...
+            self._metaclass[-1] = 'type' # XXX get the actual metaclass
+        newnode.set_line_info(newnode.last_child())
+        return newnode
+
+    def visit_assname(self, node, parent, node_name=None):
+        '''visit a node and return a AssName node'''
+        newnode = new.AssName()
+        _set_infos(node, newnode, parent)
+        newnode.name = node_name
+        self._save_assignment(newnode)
+        return newnode
+
+    def visit_augassign(self, node, parent):
+        """visit a AugAssign node by returning a fresh instance of it"""
+        newnode = new.AugAssign()
+        _lineno_parent(node, newnode, parent)
+        newnode.op = _BIN_OP_CLASSES[node.op.__class__] + "="
+        self.asscontext = "Ass"
+        newnode.target = self.visit(node.target, newnode)
+        self.asscontext = None
+        newnode.value = self.visit(node.value, newnode)
+        newnode.set_line_info(newnode.last_child())
+        return newnode
+
+    def visit_backquote(self, node, parent):
+        """visit a Backquote node by returning a fresh instance of it"""
+        newnode = new.Backquote()
+        _lineno_parent(node, newnode, parent)
+        newnode.value = self.visit(node.value, newnode)
+        newnode.set_line_info(newnode.last_child())
+        return newnode
+
+    def visit_binop(self, node, parent):
+        """visit a BinOp node by returning a fresh instance of it"""
+        newnode = new.BinOp()
+        _lineno_parent(node, newnode, parent)
+        newnode.left = self.visit(node.left, newnode)
+        newnode.right = self.visit(node.right, newnode)
+        newnode.op = _BIN_OP_CLASSES[node.op.__class__]
+        newnode.set_line_info(newnode.last_child())
+        return newnode
+
+    def visit_boolop(self, node, parent):
+        """visit a BoolOp node by returning a fresh instance of it"""
+        newnode = new.BoolOp()
+        _lineno_parent(node, newnode, parent)
+        newnode.values = [self.visit(child, newnode) for child in node.values]
+        newnode.op = _BOOL_OP_CLASSES[node.op.__class__]
+        newnode.set_line_info(newnode.last_child())
+        return newnode
+
+    def visit_break(self, node, parent):
+        """visit a Break node by returning a fresh instance of it"""
+        newnode = new.Break()
+        _set_infos(node, newnode, parent)
+        return newnode
+
+    def visit_callfunc(self, node, parent):
+        """visit a CallFunc node by returning a fresh instance of it"""
+        newnode = new.CallFunc()
+        _lineno_parent(node, newnode, parent)
+        newnode.func = self.visit(node.func, newnode)
+        newnode.args = [self.visit(child, newnode) for child in node.args]
+        if node.starargs is not None:
+            newnode.starargs = self.visit(node.starargs, newnode)
+        if node.kwargs is not None:
+            newnode.kwargs = self.visit(node.kwargs, newnode)
+        newnode.args.extend(self.visit(child, newnode) for child in node.keywords)
+        newnode.set_line_info(newnode.last_child())
+        return newnode
+
+    def visit_class(self, node, parent):
+        """visit a Class node to become astng"""
+        self._metaclass.append(self._metaclass[-1])
+        newnode = new.Class(node.name, None)
+        _lineno_parent(node, newnode, parent)
+        _init_set_doc(node, newnode)
+        newnode.bases = [self.visit(child, newnode) for child in node.bases]
+        newnode.body = [self.visit(child, newnode) for child in node.body]
+        if 'decorator_list' in node._fields and node.decorator_list:# py >= 2.6
+            newnode.decorators = self.visit_decorators(node, newnode)
+        newnode.set_line_info(newnode.last_child())
+        metaclass = self._metaclass.pop()
+        if not newnode.bases:
+            # no base classes, detect new / style old style according to
+            # current scope
+            newnode._newstyle = metaclass == 'type'
+        newnode.parent.frame().set_local(newnode.name, newnode)
+        return newnode
+
+    def visit_const(self, node, parent):
+        """visit a Const node by returning a fresh instance of it"""
+        newnode = new.Const(node.value)
+        _set_infos(node, newnode, parent)
+        return newnode
+
+    def visit_continue(self, node, parent):
+        """visit a Continue node by returning a fresh instance of it"""
+        newnode = new.Continue()
+        _set_infos(node, newnode, parent)
+        return newnode
+
+    def visit_compare(self, node, parent):
+        """visit a Compare node by returning a fresh instance of it"""
+        newnode = new.Compare()
+        _lineno_parent(node, newnode, parent)
+        newnode.left = self.visit(node.left, newnode)
+        newnode.ops = [(_CMP_OP_CLASSES[op.__class__], self.visit(expr, newnode))
+                    for (op, expr) in zip(node.ops, node.comparators)]
+        newnode.set_line_info(newnode.last_child())
+        return newnode
+
+    def visit_comprehension(self, node, parent):
+        """visit a Comprehension node by returning a fresh instance of it"""
+        newnode = new.Comprehension()
+        _lineno_parent(node, newnode, parent)
+        self.asscontext = "Ass"
+        newnode.target = self.visit(node.target, newnode)
+        self.asscontext = None
+        newnode.iter = self.visit(node.iter, newnode)
+        newnode.ifs = [self.visit(child, newnode) for child in node.ifs]
+        newnode.set_line_info(newnode.last_child())
+        return newnode
+
+    def visit_decorators(self, node, parent):
+        """visit a Decorators node by returning a fresh instance of it"""
+        # /!\ node is actually a _ast.Function node while
+        # parent is a astng.nodes.Function node
+        newnode = new.Decorators()
+        _lineno_parent(node, newnode, parent)
+        if 'decorators' in node._fields: # py < 2.6, i.e. 2.5
+            decorators = node.decorators
+        else:
+            decorators= node.decorator_list
+        newnode.nodes = [self.visit(child, newnode) for child in decorators]
+        newnode.set_line_info(newnode.last_child())
+        return newnode
+
+    def visit_delete(self, node, parent):
+        """visit a Delete node by returning a fresh instance of it"""
+        newnode = new.Delete()
+        _lineno_parent(node, newnode, parent)
+        self.asscontext = "Del"
+        newnode.targets = [self.visit(child, newnode) for child in node.targets]
+        self.asscontext = None
+        newnode.set_line_info(newnode.last_child())
+        return newnode
+
+    def visit_dict(self, node, parent):
+        """visit a Dict node by returning a fresh instance of it"""
+        newnode = new.Dict()
+        _lineno_parent(node, newnode, parent)
+        newnode.items = [(self.visit(key, newnode), self.visit(value, newnode))
+                          for key, value in zip(node.keys, node.values)]
+        newnode.set_line_info(newnode.last_child())
+        return newnode
+
+    def visit_dictcomp(self, node, parent):
+        """visit a DictComp node by returning a fresh instance of it"""
+        newnode = new.DictComp()
+        _lineno_parent(node, newnode, parent)
+        newnode.key = self.visit(node.key, newnode)
+        newnode.value = self.visit(node.value, newnode)
+        newnode.generators = [self.visit(child, newnode)
+                              for child in node.generators]
+        newnode.set_line_info(newnode.last_child())
+        return newnode
+
+    def visit_discard(self, node, parent):
+        """visit a Discard node by returning a fresh instance of it"""
+        newnode = new.Discard()
+        _lineno_parent(node, newnode, parent)
+        newnode.value = self.visit(node.value, newnode)
+        newnode.set_line_info(newnode.last_child())
+        return newnode
+
+    def visit_ellipsis(self, node, parent):
+        """visit an Ellipsis node by returning a fresh instance of it"""
+        newnode = new.Ellipsis()
+        _set_infos(node, newnode, parent)
+        return newnode
+
+    def visit_emptynode(self, node, parent):
+        """visit an EmptyNode node by returning a fresh instance of it"""
+        newnode = new.EmptyNode()
+        _set_infos(node, newnode, parent)
+        return newnode
+
+    def visit_excepthandler(self, node, parent):
+        """visit an ExceptHandler node by returning a fresh instance of it"""
+        newnode = new.ExceptHandler()
+        _lineno_parent(node, newnode, parent)
+        if node.type is not None:
+            newnode.type = self.visit(node.type, newnode)
+        if node.name is not None:
+            # /!\ node.name can be a tuple
+            self.asscontext = "Ass"
+            newnode.name = self.visit(node.name, newnode)
+            self.asscontext = None
+        newnode.body = [self.visit(child, newnode) for child in node.body]
+        newnode.set_line_info(newnode.last_child())
+        return newnode
+
+    def visit_exec(self, node, parent):
+        """visit an Exec node by returning a fresh instance of it"""
+        newnode = new.Exec()
+        _lineno_parent(node, newnode, parent)
+        newnode.expr = self.visit(node.body, newnode)
+        if node.globals is not None:
+            newnode.globals = self.visit(node.globals, newnode)
+        if node.locals is not None:
+            newnode.locals = self.visit(node.locals, newnode)
+        newnode.set_line_info(newnode.last_child())
+        return newnode
+
+    def visit_extslice(self, node, parent):
+        """visit an ExtSlice node by returning a fresh instance of it"""
+        newnode = new.ExtSlice()
+        _lineno_parent(node, newnode, parent)
+        newnode.dims = [self.visit(dim, newnode) for dim in node.dims]
+        newnode.set_line_info(newnode.last_child())
+        return newnode
+
+    def visit_for(self, node, parent):
+        """visit a For node by returning a fresh instance of it"""
+        newnode = new.For()
+        _lineno_parent(node, newnode, parent)
+        self.asscontext = "Ass"
+        newnode.target = self.visit(node.target, newnode)
+        self.asscontext = None
+        newnode.iter = self.visit(node.iter, newnode)
+        newnode.body = [self.visit(child, newnode) for child in node.body]
+        newnode.orelse = [self.visit(child, newnode) for child in node.orelse]
+        newnode.set_line_info(newnode.last_child())
+        return newnode
+
+    def visit_from(self, node, parent):
+        """visit a From node by returning a fresh instance of it"""
+        names = [(alias.name, alias.asname) for alias in node.names]
+        newnode = new.From(node.module or '', names, node.level)
+        _set_infos(node, newnode, parent)
+        # store From names to add them to locals after building
+        self._from_nodes.append(newnode)
+        return newnode
+
+    def visit_function(self, node, parent):
+        """visit an Function node to become astng"""
+        self._global_names.append({})
+        newnode = new.Function(node.name, None)
+        _lineno_parent(node, newnode, parent)
+        _init_set_doc(node, newnode)
+        newnode.args = self.visit(node.args, newnode)
+        newnode.body = [self.visit(child, newnode) for child in node.body]
+        if 'decorators' in node._fields: # py < 2.6
+            attr = 'decorators'
+        else:
+            attr = 'decorator_list'
+        decorators = getattr(node, attr)
+        if decorators:
+            newnode.decorators = self.visit_decorators(node, newnode)
+        newnode.set_line_info(newnode.last_child())
+        self._global_names.pop()
+        frame = newnode.parent.frame()
+        if isinstance(frame, new.Class):
+            if newnode.name == '__new__':
+                newnode.type = 'classmethod'
+            else:
+                newnode.type = 'method'
+        if newnode.decorators is not None:
+            for decorator_expr in newnode.decorators.nodes:
+                if isinstance(decorator_expr, new.Name):
+                    if decorator_expr.name in ('classmethod', 'staticmethod'):
+                        newnode.type = decorator_expr.name
+                    elif decorator_expr.name == 'classproperty':
+                        newnode.type = 'classmethod'
+        frame.set_local(newnode.name, newnode)
+        return newnode
+
+    def visit_genexpr(self, node, parent):
+        """visit a GenExpr node by returning a fresh instance of it"""
+        newnode = new.GenExpr()
+        _lineno_parent(node, newnode, parent)
+        newnode.elt = self.visit(node.elt, newnode)
+        newnode.generators = [self.visit(child, newnode) for child in node.generators]
+        newnode.set_line_info(newnode.last_child())
+        return newnode
+
+    def visit_getattr(self, node, parent):
+        """visit a Getattr node by returning a fresh instance of it"""
+        if self.asscontext == "Del":
+            # FIXME : maybe we should reintroduce and visit_delattr ?
+            # for instance, deactivating asscontext
+            newnode = new.DelAttr()
+        elif self.asscontext == "Ass":
+            # FIXME : maybe we should call visit_assattr ?
+            newnode = new.AssAttr()
+            self._delayed_assattr.append(newnode)
+        else:
+            newnode = new.Getattr()
+        _lineno_parent(node, newnode, parent)
+        asscontext, self.asscontext = self.asscontext, None
+        newnode.expr = self.visit(node.value, newnode)
+        self.asscontext = asscontext
+        newnode.attrname = node.attr
+        newnode.set_line_info(newnode.last_child())
+        return newnode
+
+    def visit_global(self, node, parent):
+        """visit an Global node to become astng"""
+        newnode = new.Global(node.names)
+        _set_infos(node, newnode, parent)
+        if self._global_names: # global at the module level, no effect
+            for name in node.names:
+                self._global_names[-1].setdefault(name, []).append(newnode)
+        return newnode
+
+    def visit_if(self, node, parent):
+        """visit a If node by returning a fresh instance of it"""
+        newnode = new.If()
+        _lineno_parent(node, newnode, parent)
+        newnode.test = self.visit(node.test, newnode)
+        newnode.body = [self.visit(child, newnode) for child in node.body]
+        newnode.orelse = [self.visit(child, newnode) for child in node.orelse]
+        newnode.set_line_info(newnode.last_child())
+        return newnode
+
+    def visit_ifexp(self, node, parent):
+        """visit a IfExp node by returning a fresh instance of it"""
+        newnode = new.IfExp()
+        _lineno_parent(node, newnode, parent)
+        newnode.test = self.visit(node.test, newnode)
+        newnode.body = self.visit(node.body, newnode)
+        newnode.orelse = self.visit(node.orelse, newnode)
+        newnode.set_line_info(newnode.last_child())
+        return newnode
+
+    def visit_import(self, node, parent):
+        """visit a Import node by returning a fresh instance of it"""
+        newnode = new.Import()
+        _set_infos(node, newnode, parent)
+        newnode.names = [(alias.name, alias.asname) for alias in node.names]
+        # save import names in parent's locals:
+        for (name, asname) in newnode.names:
+            name = asname or name
+            newnode.parent.set_local(name.split('.')[0], newnode)
+        return newnode
+
+    def visit_index(self, node, parent):
+        """visit a Index node by returning a fresh instance of it"""
+        newnode = new.Index()
+        _lineno_parent(node, newnode, parent)
+        newnode.value = self.visit(node.value, newnode)
+        newnode.set_line_info(newnode.last_child())
+        return newnode
+
+    def visit_keyword(self, node, parent):
+        """visit a Keyword node by returning a fresh instance of it"""
+        newnode = new.Keyword()
+        _lineno_parent(node, newnode, parent)
+        newnode.arg = node.arg
+        newnode.value = self.visit(node.value, newnode)
+        newnode.set_line_info(newnode.last_child())
+        return newnode
+
+    def visit_lambda(self, node, parent):
+        """visit a Lambda node by returning a fresh instance of it"""
+        newnode = new.Lambda()
+        _lineno_parent(node, newnode, parent)
+        newnode.args = self.visit(node.args, newnode)
+        newnode.body = self.visit(node.body, newnode)
+        newnode.set_line_info(newnode.last_child())
+        return newnode
+
+    def visit_list(self, node, parent):
+        """visit a List node by returning a fresh instance of it"""
+        newnode = new.List()
+        _lineno_parent(node, newnode, parent)
+        newnode.elts = [self.visit(child, newnode) for child in node.elts]
+        newnode.set_line_info(newnode.last_child())
+        return newnode
+
+    def visit_listcomp(self, node, parent):
+        """visit a ListComp node by returning a fresh instance of it"""
+        newnode = new.ListComp()
+        _lineno_parent(node, newnode, parent)
+        newnode.elt = self.visit(node.elt, newnode)
+        newnode.generators = [self.visit(child, newnode)
+                              for child in node.generators]
+        newnode.set_line_info(newnode.last_child())
+        return newnode
+
+    def visit_module(self, node, modname, package):
+        """visit a Module node by returning a fresh instance of it"""
+        newnode = new.Module(modname, None)
+        newnode.package = package
+        _lineno_parent(node, newnode, parent=None)
+        _init_set_doc(node, newnode)
+        newnode.body = [self.visit(child, newnode) for child in node.body]
+        newnode.set_line_info(newnode.last_child())
+        return newnode
+
+    def visit_name(self, node, parent):
+        """visit a Name node by returning a fresh instance of it"""
+        # True and False can be assigned to something in py2x, so we have to
+        # check first the asscontext
+        if self.asscontext == "Del":
+            newnode = new.DelName()
+        elif self.asscontext is not None: # Ass
+            assert self.asscontext == "Ass"
+            newnode = new.AssName()
+        elif node.id in CONST_NAME_TRANSFORMS:
+            newnode = new.Const(CONST_NAME_TRANSFORMS[node.id])
+            _set_infos(node, newnode, parent)
+            return newnode
+        else:
+            newnode = new.Name()
+        _lineno_parent(node, newnode, parent)
+        newnode.name = node.id
+        # XXX REMOVE me :
+        if self.asscontext in ('Del', 'Ass'): # 'Aug' ??
+            self._save_assignment(newnode)
+        newnode.set_line_info(newnode.last_child())
+        return newnode
+
+    def visit_bytes(self, node, parent):
+        """visit a Bytes node by returning a fresh instance of Const"""
+        newnode = new.Const(node.s)
+        _set_infos(node, newnode, parent)
+        return newnode
+
+    def visit_num(self, node, parent):
+        """visit a Num node by returning a fresh instance of Const"""
+        newnode = new.Const(node.n)
+        _set_infos(node, newnode, parent)
+        return newnode
+
+    def visit_pass(self, node, parent):
+        """visit a Pass node by returning a fresh instance of it"""
+        newnode = new.Pass()
+        _set_infos(node, newnode, parent)
+        return newnode
+
+    def visit_str(self, node, parent):
+        """visit a Str node by returning a fresh instance of Const"""
+        newnode = new.Const(node.s)
+        _set_infos(node, newnode, parent)
+        return newnode
+
+    def visit_print(self, node, parent):
+        """visit a Print node by returning a fresh instance of it"""
+        newnode = new.Print()
+        _lineno_parent(node, newnode, parent)
+        newnode.nl = node.nl
+        if node.dest is not None:
+            newnode.dest = self.visit(node.dest, newnode)
+        newnode.values = [self.visit(child, newnode) for child in node.values]
+        newnode.set_line_info(newnode.last_child())
+        return newnode
+
+    def visit_raise(self, node, parent):
+        """visit a Raise node by returning a fresh instance of it"""
+        newnode = new.Raise()
+        _lineno_parent(node, newnode, parent)
+        if node.type is not None:
+            newnode.exc = self.visit(node.type, newnode)
+        if node.inst is not None:
+            newnode.inst = self.visit(node.inst, newnode)
+        if node.tback is not None:
+            newnode.tback = self.visit(node.tback, newnode)
+        newnode.set_line_info(newnode.last_child())
+        return newnode
+
+    def visit_return(self, node, parent):
+        """visit a Return node by returning a fresh instance of it"""
+        newnode = new.Return()
+        _lineno_parent(node, newnode, parent)
+        if node.value is not None:
+            newnode.value = self.visit(node.value, newnode)
+        newnode.set_line_info(newnode.last_child())
+        return newnode
+
+    def visit_set(self, node, parent):
+        """visit a Tuple node by returning a fresh instance of it"""
+        newnode = new.Set()
+        _lineno_parent(node, newnode, parent)
+        newnode.elts = [self.visit(child, newnode) for child in node.elts]
+        newnode.set_line_info(newnode.last_child())
+        return newnode
+
+    def visit_setcomp(self, node, parent):
+        """visit a SetComp node by returning a fresh instance of it"""
+        newnode = new.SetComp()
+        _lineno_parent(node, newnode, parent)
+        newnode.elt = self.visit(node.elt, newnode)
+        newnode.generators = [self.visit(child, newnode)
+                              for child in node.generators]
+        newnode.set_line_info(newnode.last_child())
+        return newnode
+
+    def visit_slice(self, node, parent):
+        """visit a Slice node by returning a fresh instance of it"""
+        newnode = new.Slice()
+        _lineno_parent(node, newnode, parent)
+        if node.lower is not None:
+            newnode.lower = self.visit(node.lower, newnode)
+        if node.upper is not None:
+            newnode.upper = self.visit(node.upper, newnode)
+        if node.step is not None:
+            newnode.step = self.visit(node.step, newnode)
+        newnode.set_line_info(newnode.last_child())
+        return newnode
+
+    def visit_subscript(self, node, parent):
+        """visit a Subscript node by returning a fresh instance of it"""
+        newnode = new.Subscript()
+        _lineno_parent(node, newnode, parent)
+        subcontext, self.asscontext = self.asscontext, None
+        newnode.value = self.visit(node.value, newnode)
+        newnode.slice = self.visit(node.slice, newnode)
+        self.asscontext = subcontext
+        newnode.set_line_info(newnode.last_child())
+        return newnode
+
+    def visit_tryexcept(self, node, parent):
+        """visit a TryExcept node by returning a fresh instance of it"""
+        newnode = new.TryExcept()
+        _lineno_parent(node, newnode, parent)
+        newnode.body = [self.visit(child, newnode) for child in node.body]
+        newnode.handlers = [self.visit(child, newnode) for child in node.handlers]
+        newnode.orelse = [self.visit(child, newnode) for child in node.orelse]
+        newnode.set_line_info(newnode.last_child())
+        return newnode
+
+    def visit_tryfinally(self, node, parent):
+        """visit a TryFinally node by returning a fresh instance of it"""
+        newnode = new.TryFinally()
+        _lineno_parent(node, newnode, parent)
+        newnode.body = [self.visit(child, newnode) for child in node.body]
+        newnode.finalbody = [self.visit(n, newnode) for n in node.finalbody]
+        newnode.set_line_info(newnode.last_child())
+        return newnode
+
+    def visit_tuple(self, node, parent):
+        """visit a Tuple node by returning a fresh instance of it"""
+        newnode = new.Tuple()
+        _lineno_parent(node, newnode, parent)
+        newnode.elts = [self.visit(child, newnode) for child in node.elts]
+        newnode.set_line_info(newnode.last_child())
+        return newnode
+
+    def visit_unaryop(self, node, parent):
+        """visit a UnaryOp node by returning a fresh instance of it"""
+        newnode = new.UnaryOp()
+        _lineno_parent(node, newnode, parent)
+        newnode.operand = self.visit(node.operand, newnode)
+        newnode.op = _UNARY_OP_CLASSES[node.op.__class__]
+        newnode.set_line_info(newnode.last_child())
+        return newnode
+
+    def visit_while(self, node, parent):
+        """visit a While node by returning a fresh instance of it"""
+        newnode = new.While()
+        _lineno_parent(node, newnode, parent)
+        newnode.test = self.visit(node.test, newnode)
+        newnode.body = [self.visit(child, newnode) for child in node.body]
+        newnode.orelse = [self.visit(child, newnode) for child in node.orelse]
+        newnode.set_line_info(newnode.last_child())
+        return newnode
+
+    def visit_with(self, node, parent):
+        """visit a With node by returning a fresh instance of it"""
+        newnode = new.With()
+        _lineno_parent(node, newnode, parent)
+        newnode.expr = self.visit(node.context_expr, newnode)
+        self.asscontext = "Ass"
+        if node.optional_vars is not None:
+            newnode.vars = self.visit(node.optional_vars, newnode)
+        self.asscontext = None
+        newnode.body = [self.visit(child, newnode) for child in node.body]
+        newnode.set_line_info(newnode.last_child())
+        return newnode
+
+    def visit_yield(self, node, parent):
+        """visit a Yield node by returning a fresh instance of it"""
+        newnode = new.Yield()
+        _lineno_parent(node, newnode, parent)
+        if node.value is not None:
+            newnode.value = self.visit(node.value, newnode)
+        newnode.set_line_info(newnode.last_child())
+        return newnode
+
+
+class TreeRebuilder3k(TreeRebuilder):
+    """extend and overwrite TreeRebuilder for python3k"""
+
+    def visit_arg(self, node, parent):
+        """visit a arg node by returning a fresh AssName instance"""
+        # the <arg> node is coming from py>=3.0, but we use AssName in py2.x
+        # XXX or we should instead introduce a Arg node in astng ?
+        return self.visit_assname(node, parent, node.arg)
+
+    def visit_excepthandler(self, node, parent):
+        """visit an ExceptHandler node by returning a fresh instance of it"""
+        newnode = new.ExceptHandler()
+        _lineno_parent(node, newnode, parent)
+        if node.type is not None:
+            newnode.type = self.visit(node.type, newnode)
+        if node.name is not None:
+            newnode.name = self.visit_assname(node, newnode, node.name)
+        newnode.body = [self.visit(child, newnode) for child in node.body]
+        newnode.set_line_info(newnode.last_child())
+        return newnode
+
+    def visit_nonlocal(self, node, parent):
+        """visit a Nonlocal node and return a new instance of it"""
+        newnode = new.Nonlocal(node.names)
+        _set_infos(node, newnode, parent)
+        return newnode
+
+    def visit_raise(self, node, parent):
+        """visit a Raise node by returning a fresh instance of it"""
+        newnode = new.Raise()
+        _lineno_parent(node, newnode, parent)
+        # no traceback; anyway it is not used in Pylint
+        if node.exc is not None:
+            newnode.exc = self.visit(node.exc, newnode)
+        if node.cause is not None:
+            newnode.cause = self.visit(node.cause, newnode)
+        newnode.set_line_info(newnode.last_child())
+        return newnode
+
+    def visit_starred(self, node, parent):
+        """visit a Starred node and return a new instance of it"""
+        newnode = new.Starred()
+        _lineno_parent(node, newnode, parent)
+        newnode.value = self.visit(node.value, newnode)
+        newnode.set_line_info(newnode.last_child())
+        return newnode
+
+
+if sys.version_info >= (3, 0):
+    TreeRebuilder = TreeRebuilder3k
+
+

pylibs/logilab/astng/scoped_nodes.py

@@ -0,0 +1,977 @@
+# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+# copyright 2003-2010 Sylvain Thenault, all rights reserved.
+# contact mailto:thenault@gmail.com
+#
+# This file is part of logilab-astng.
+#
+# logilab-astng is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Lesser General Public License as published by the
+# Free Software Foundation, either version 2.1 of the License, or (at your
+# option) any later version.
+#
+# logilab-astng is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License
+# for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with logilab-astng. If not, see <http://www.gnu.org/licenses/>.
+"""This module contains the classes for "scoped" node, i.e. which are opening a
+new local scope in the language definition : Module, Class, Function (and
+Lambda, GenExpr, DictComp and SetComp to some extent).
+"""
+from __future__ import with_statement
+
+__doctype__ = "restructuredtext en"
+
+import sys
+from itertools import chain
+
+from logilab.common.compat import builtins
+from logilab.common.decorators import cached
+
+from logilab.astng import BUILTINS_MODULE
+from logilab.astng.exceptions import NotFoundError, NoDefault, \
+     ASTNGBuildingException, InferenceError
+from logilab.astng.node_classes import Const, DelName, DelAttr, \
+     Dict, From, List, Name, Pass, Raise, Return, Tuple, Yield, \
+     are_exclusive, LookupMixIn, const_factory as cf, unpack_infer
+from logilab.astng.bases import NodeNG, InferenceContext, Instance,\
+     YES, Generator, UnboundMethod, BoundMethod, _infer_stmts, copy_context, \
+     BUILTINS_NAME
+from logilab.astng.mixins import FilterStmtsMixin
+from logilab.astng.bases import Statement
+from logilab.astng.manager import ASTNGManager
+
+
+def remove_nodes(func, cls):
+    def wrapper(*args, **kwargs):
+        nodes = [n for n in func(*args, **kwargs) if not isinstance(n, cls)]
+        if not nodes:
+            raise NotFoundError()
+        return nodes
+    return wrapper
+
+
+def function_to_method(n, klass):
+    if isinstance(n, Function):
+        if n.type == 'classmethod':
+            return BoundMethod(n, klass)
+        if n.type != 'staticmethod':
+            return UnboundMethod(n)
+    return n
+
+def std_special_attributes(self, name, add_locals=True):
+    if add_locals:
+        locals = self.locals
+    else:
+        locals = {}
+    if name == '__name__':
+        return [cf(self.name)] + locals.get(name, [])
+    if name == '__doc__':
+        return [cf(self.doc)] + locals.get(name, [])
+    if name == '__dict__':
+        return [Dict()] + locals.get(name, [])
+    raise NotFoundError(name)
+
+MANAGER = ASTNGManager()
+def builtin_lookup(name):
+    """lookup a name into the builtin module
+    return the list of matching statements and the astng for the builtin
+    module
+    """
+    builtin_astng = MANAGER.astng_from_module(builtins)
+    if name == '__dict__':
+        return builtin_astng, ()
+    try:
+        stmts = builtin_astng.locals[name]
+    except KeyError:
+        stmts = ()
+    return builtin_astng, stmts
+
+
+# TODO move this Mixin to mixins.py; problem: 'Function' in _scope_lookup
+class LocalsDictNodeNG(LookupMixIn, NodeNG):
+    """ this class provides locals handling common to Module, Function
+    and Class nodes, including a dict like interface for direct access
+    to locals information
+    """
+
+    # attributes below are set by the builder module or by raw factories
+
+    # dictionary of locals with name as key and node defining the local as
+    # value
+
+    def qname(self):
+        """return the 'qualified' name of the node, eg module.name,
+        module.class.name ...
+        """
+        if self.parent is None:
+            return self.name
+        return '%s.%s' % (self.parent.frame().qname(), self.name)
+
+    def frame(self):
+        """return the first parent frame node (i.e. Module, Function or Class)
+        """
+        return self
+
+    def scope(self):
+        """return the first node defining a new scope (i.e. Module,
+        Function, Class, Lambda but also GenExpr, DictComp and SetComp)
+        """
+        return self
+
+
+    def _scope_lookup(self, node, name, offset=0):
+        """XXX method for interfacing the scope lookup"""
+        try:
+            stmts = node._filter_stmts(self.locals[name], self, offset)
+        except KeyError:
+            stmts = ()
+        if stmts:
+            return self, stmts
+        if self.parent: # i.e. not Module
+            # nested scope: if parent scope is a function, that's fine
+            # else jump to the module
+            pscope = self.parent.scope()
+            if not pscope.is_function:
+                pscope = pscope.root()
+            return pscope.scope_lookup(node, name)
+        return builtin_lookup(name) # Module
+
+
+
+    def set_local(self, name, stmt):
+        """define <name> in locals (<stmt> is the node defining the name)
+        if the node is a Module node (i.e. has globals), add the name to
+        globals
+
+        if the name is already defined, ignore it
+        """
+        #assert not stmt in self.locals.get(name, ()), (self, stmt)
+        self.locals.setdefault(name, []).append(stmt)
+
+    __setitem__ = set_local
+
+    def _append_node(self, child):
+        """append a child, linking it in the tree"""
+        self.body.append(child)
+        child.parent = self
+
+    def add_local_node(self, child_node, name=None):
+        """append a child which should alter locals to the given node"""
+        if name != '__class__':
+            # add __class__ node as a child will cause infinite recursion later!
+            self._append_node(child_node)
+        self.set_local(name or child_node.name, child_node)
+
+
+    def __getitem__(self, item):
+        """method from the `dict` interface returning the first node
+        associated with the given name in the locals dictionary
+
+        :type item: str
+        :param item: the name of the locally defined object
+        :raises KeyError: if the name is not defined
+        """
+        return self.locals[item][0]
+
+    def __iter__(self):
+        """method from the `dict` interface returning an iterator on
+        `self.keys()`
+        """
+        return iter(self.keys())
+
+    def keys(self):
+        """method from the `dict` interface returning a tuple containing
+        locally defined names
+        """
+        return self.locals.keys()
+
+    def values(self):
+        """method from the `dict` interface returning a tuple containing
+        locally defined nodes which are instance of `Function` or `Class`
+        """
+        return [self[key] for key in self.keys()]
+
+    def items(self):
+        """method from the `dict` interface returning a list of tuple
+        containing each locally defined name with its associated node,
+        which is an instance of `Function` or `Class`
+        """
+        return zip(self.keys(), self.values())
+
+
+    def __contains__(self, name):
+        return name in self.locals
+    has_key = __contains__
+
+# Module  #####################################################################
+
+class Module(LocalsDictNodeNG):
+    _astng_fields = ('body',)
+
+    fromlineno = 0
+    lineno = 0
+
+    # attributes below are set by the builder module or by raw factories
+
+    # the file from which as been extracted the astng representation. It may
+    # be None if the representation has been built from a built-in module
+    file = None
+    # the module name
+    name = None
+    # boolean for astng built from source (i.e. ast)
+    pure_python = None
+    # boolean for package module
+    package = None
+    # dictionary of globals with name as key and node defining the global
+    # as value
+    globals = None
+
+    # names of python special attributes (handled by getattr impl.)
+    special_attributes = set(('__name__', '__doc__', '__file__', '__path__',
+                              '__dict__'))
+    # names of module attributes available through the global scope
+    scope_attrs = set(('__name__', '__doc__', '__file__', '__path__'))
+
+    def __init__(self, name, doc, pure_python=True):
+        self.name = name
+        self.doc = doc
+        self.pure_python = pure_python
+        self.locals = self.globals = {}
+        self.body = []
+
+    @property
+    def file_stream(self):
+        if self.file is not None:
+            return open(self.file)
+        return None
+
+    def block_range(self, lineno):
+        """return block line numbers.
+
+        start from the beginning whatever the given lineno
+        """
+        return self.fromlineno, self.tolineno
+
+    def scope_lookup(self, node, name, offset=0):
+        if name in self.scope_attrs and not name in self.locals:
+            try:
+                return self, self.getattr(name)
+            except NotFoundError:
+                return self, ()
+        return self._scope_lookup(node, name, offset)
+
+    def pytype(self):
+        return '%s.module' % BUILTINS_MODULE
+
+    def display_type(self):
+        return 'Module'
+
+    def getattr(self, name, context=None, ignore_locals=False):
+        if name in self.special_attributes:
+            if name == '__file__':
+                return [cf(self.file)] + self.locals.get(name, [])
+            if name == '__path__' and self.package:
+                return [List()] + self.locals.get(name, [])
+            return std_special_attributes(self, name)
+        if not ignore_locals and name in self.locals:
+            return self.locals[name]
+        if self.package:
+            try:
+                return [self.import_module(name, relative_only=True)]
+            except ASTNGBuildingException:
+                raise NotFoundError(name)
+            except Exception:# XXX pylint tests never pass here; do we need it?
+                import traceback
+                traceback.print_exc()
+        raise NotFoundError(name)
+    getattr = remove_nodes(getattr, DelName)
+
+    def igetattr(self, name, context=None):
+        """inferred getattr"""
+        # set lookup name since this is necessary to infer on import nodes for
+        # instance
+        context = copy_context(context)
+        context.lookupname = name
+        try:
+            return _infer_stmts(self.getattr(name, context), context, frame=self)
+        except NotFoundError:
+            raise InferenceError(name)
+
+    def fully_defined(self):
+        """return True if this module has been built from a .py file
+        and so contains a complete representation including the code
+        """
+        return self.file is not None and self.file.endswith('.py')
+
+    def statement(self):
+        """return the first parent node marked as statement node
+        consider a module as a statement...
+        """
+        return self
+
+    def previous_sibling(self):
+        """module has no sibling"""
+        return
+
+    def next_sibling(self):
+        """module has no sibling"""
+        return
+
+    if sys.version_info < (2, 8):
+        def absolute_import_activated(self):
+            for stmt in self.locals.get('absolute_import', ()):
+                if isinstance(stmt, From) and stmt.modname == '__future__':
+                    return True
+            return False
+    else:
+        absolute_import_activated = lambda self: True
+
+    def import_module(self, modname, relative_only=False, level=None):
+        """import the given module considering self as context"""
+        if relative_only and level is None:
+            level = 0
+        absmodname = self.relative_to_absolute_name(modname, level)
+        try:
+            return MANAGER.astng_from_module_name(absmodname)
+        except ASTNGBuildingException:
+            # we only want to import a sub module or package of this module,
+            # skip here
+            if relative_only:
+                raise
+        return MANAGER.astng_from_module_name(modname)
+
+    def relative_to_absolute_name(self, modname, level):
+        """return the absolute module name for a relative import.
+
+        The relative import can be implicit or explicit.
+        """
+        # XXX this returns non sens when called on an absolute import
+        # like 'pylint.checkers.logilab.astng.utils'
+        # XXX doesn't return absolute name if self.name isn't absolute name
+        if self.absolute_import_activated() and level is None:
+            return modname
+        if level:
+            if self.package:
+                level = level - 1
+            package_name = self.name.rsplit('.', level)[0]
+        elif self.package:
+            package_name = self.name
+        else:
+            package_name = self.name.rsplit('.', 1)[0]
+        if package_name:
+            if not modname:
+                return package_name
+            return '%s.%s' % (package_name, modname)
+        return modname
+
+
+    def wildcard_import_names(self):
+        """return the list of imported names when this module is 'wildcard
+        imported'
+
+        It doesn't include the '__builtins__' name which is added by the
+        current CPython implementation of wildcard imports.
+        """
+        # take advantage of a living module if it exists
+        try:
+            living = sys.modules[self.name]
+        except KeyError:
+            pass
+        else:
+            try:
+                return living.__all__
+            except AttributeError:
+                return [name for name in living.__dict__.keys()
+                        if not name.startswith('_')]
+        # else lookup the astng
+        #
+        # We separate the different steps of lookup in try/excepts
+        # to avoid catching too many Exceptions
+        # However, we can not analyse dynamically constructed __all__
+        try:
+            all = self['__all__']
+        except KeyError:
+            return [name for name in self.keys() if not name.startswith('_')]
+        try:
+            explicit = all.assigned_stmts().next()
+        except InferenceError:
+            return [name for name in self.keys() if not name.startswith('_')]
+        except AttributeError:
+            # not an assignment node
+            # XXX infer?
+            return [name for name in self.keys() if not name.startswith('_')]
+        try:
+            # should be a Tuple/List of constant string / 1 string not allowed
+            return [const.value for const in explicit.elts]
+        except AttributeError:
+            return [name for name in self.keys() if not name.startswith('_')]
+
+
+class ComprehensionScope(LocalsDictNodeNG):
+    def frame(self):
+        return self.parent.frame()
+
+    scope_lookup = LocalsDictNodeNG._scope_lookup
+
+
+class GenExpr(ComprehensionScope):
+    _astng_fields = ('elt', 'generators')
+
+    def __init__(self):
+        self.locals = {}
+        self.elt = None
+        self.generators = []
+
+
+class DictComp(ComprehensionScope):
+    _astng_fields = ('key', 'value', 'generators')
+
+    def __init__(self):
+        self.locals = {}
+        self.key = None
+        self.value = None
+        self.generators = []
+
+
+class SetComp(ComprehensionScope):
+    _astng_fields = ('elt', 'generators')
+
+    def __init__(self):
+        self.locals = {}
+        self.elt = None
+        self.generators = []
+
+
+class _ListComp(NodeNG):
+    """class representing a ListComp node"""
+    _astng_fields = ('elt', 'generators')
+    elt = None
+    generators = None
+
+if sys.version_info >= (3, 0):
+    class ListComp(_ListComp, ComprehensionScope):
+        """class representing a ListComp node"""
+        def __init__(self):
+            self.locals = {}
+else:
+    class ListComp(_ListComp):
+        """class representing a ListComp node"""
+
+# Function  ###################################################################
+
+
+class Lambda(LocalsDictNodeNG, FilterStmtsMixin):
+    _astng_fields = ('args', 'body',)
+    name = '<lambda>'
+
+    # function's type, 'function' | 'method' | 'staticmethod' | 'classmethod'
+    type = 'function'
+
+    def __init__(self):
+        self.locals = {}
+        self.args = []
+        self.body = []
+
+    def pytype(self):
+        if 'method' in self.type:
+            return '%s.instancemethod' % BUILTINS_MODULE
+        return '%s.function' % BUILTINS_MODULE
+
+    def display_type(self):
+        if 'method' in self.type:
+            return 'Method'
+        return 'Function'
+
+    def callable(self):
+        return True
+
+    def argnames(self):
+        """return a list of argument names"""
+        if self.args.args: # maybe None with builtin functions
+            names = _rec_get_names(self.args.args)
+        else:
+            names = []
+        if self.args.vararg:
+            names.append(self.args.vararg)
+        if self.args.kwarg:
+            names.append(self.args.kwarg)
+        return names
+
+    def infer_call_result(self, caller, context=None):
+        """infer what a function is returning when called"""
+        return self.body.infer(context)
+
+    def scope_lookup(self, node, name, offset=0):
+        if node in self.args.defaults:
+            frame = self.parent.frame()
+            # line offset to avoid that def func(f=func) resolve the default
+            # value to the defined function
+            offset = -1
+        else:
+            # check this is not used in function decorators
+            frame = self
+        return frame._scope_lookup(node, name, offset)
+
+
+class Function(Statement, Lambda):
+    _astng_fields = ('decorators', 'args', 'body')
+
+    special_attributes = set(('__name__', '__doc__', '__dict__'))
+    is_function = True
+    # attributes below are set by the builder module or by raw factories
+    blockstart_tolineno = None
+    decorators = None
+
+    def __init__(self, name, doc):
+        self.locals = {}
+        self.args = []
+        self.body = []
+        self.decorators = None
+        self.name = name
+        self.doc = doc
+        self.extra_decorators = []
+        self.instance_attrs = {}
+
+    def set_line_info(self, lastchild):
+        self.fromlineno = self.lineno
+        # lineno is the line number of the first decorator, we want the def statement lineno
+        if self.decorators is not None:
+            self.fromlineno += len(self.decorators.nodes)
+        self.tolineno = lastchild.tolineno
+        self.blockstart_tolineno = self.args.tolineno
+
+    def block_range(self, lineno):
+        """return block line numbers.
+
+        start from the "def" position whatever the given lineno
+        """
+        return self.fromlineno, self.tolineno
+
+    def getattr(self, name, context=None):
+        """this method doesn't look in the instance_attrs dictionary since it's
+        done by an Instance proxy at inference time.
+        """
+        if name == '__module__':
+            return [cf(self.root().qname())]
+        if name in self.instance_attrs:
+            return self.instance_attrs[name]
+        return std_special_attributes(self, name, False)
+
+    def is_method(self):
+        """return true if the function node should be considered as a method"""
+        # check we are defined in a Class, because this is usually expected
+        # (e.g. pylint...) when is_method() return True
+        return self.type != 'function' and isinstance(self.parent.frame(), Class)
+
+    def decoratornames(self):
+        """return a list of decorator qualified names"""
+        result = set()
+        decoratornodes = []
+        if self.decorators is not None:
+            decoratornodes += self.decorators.nodes
+        decoratornodes += self.extra_decorators
+        for decnode in decoratornodes:
+            for infnode in decnode.infer():
+                result.add(infnode.qname())
+        return result
+    decoratornames = cached(decoratornames)
+
+    def is_bound(self):
+        """return true if the function is bound to an Instance or a class"""
+        return self.type == 'classmethod'
+
+    def is_abstract(self, pass_is_abstract=True):
+        """return true if the method is abstract
+        It's considered as abstract if the only statement is a raise of
+        NotImplementError, or, if pass_is_abstract, a pass statement
+        """
+        for child_node in self.body:
+            if isinstance(child_node, Raise):
+                if child_node.raises_not_implemented():
+                    return True
+            if pass_is_abstract and isinstance(child_node, Pass):
+                return True
+            return False
+        # empty function is the same as function with a single "pass" statement
+        if pass_is_abstract:
+            return True
+
+    def is_generator(self):
+        """return true if this is a generator function"""
+        # XXX should be flagged, not computed
+        try:
+            return self.nodes_of_class(Yield, skip_klass=Function).next()
+        except StopIteration:
+            return False
+
+    def infer_call_result(self, caller, context=None):
+        """infer what a function is returning when called"""
+        if self.is_generator():
+            yield Generator(self)
+            return
+        returns = self.nodes_of_class(Return, skip_klass=Function)
+        for returnnode in returns:
+            if returnnode.value is None:
+                yield Const(None)
+            else:
+                try:
+                    for infered in returnnode.value.infer(context):
+                        yield infered
+                except InferenceError:
+                    yield YES
+
+
+def _rec_get_names(args, names=None):
+    """return a list of all argument names"""
+    if names is None:
+        names = []
+    for arg in args:
+        if isinstance(arg, Tuple):
+            _rec_get_names(arg.elts, names)
+        else:
+            names.append(arg.name)
+    return names
+
+
+# Class ######################################################################
+
+def _class_type(klass, ancestors=None):
+    """return a Class node type to differ metaclass, interface and exception
+    from 'regular' classes
+    """
+    # XXX we have to store ancestors in case we have a ancestor loop
+    if klass._type is not None:
+        return klass._type
+    if klass.name == 'type':
+        klass._type = 'metaclass'
+    elif klass.name.endswith('Interface'):
+        klass._type = 'interface'
+    elif klass.name.endswith('Exception'):
+        klass._type = 'exception'
+    else:
+        if ancestors is None:
+            ancestors = set()
+        if klass in ancestors:
+            # XXX we are in loop ancestors, and have found no type
+            klass._type = 'class'
+            return 'class'
+        ancestors.add(klass)
+        # print >> sys.stderr, '_class_type', repr(klass)
+        for base in klass.ancestors(recurs=False):
+            if _class_type(base, ancestors) != 'class':
+                klass._type = base.type
+                break
+    if klass._type is None:
+        klass._type = 'class'
+    return klass._type
+
+def _iface_hdlr(iface_node):
+    """a handler function used by interfaces to handle suspicious
+    interface nodes
+    """
+    return True
+
+
+class Class(Statement, LocalsDictNodeNG, FilterStmtsMixin):
+
+    # some of the attributes below are set by the builder module or
+    # by a raw factories
+
+    # a dictionary of class instances attributes
+    _astng_fields = ('decorators', 'bases', 'body') # name
+
+    decorators = None
+    special_attributes = set(('__name__', '__doc__', '__dict__', '__module__',
+                              '__bases__', '__mro__', '__subclasses__'))
+    blockstart_tolineno = None
+
+    _type = None
+    type = property(_class_type,
+                    doc="class'type, possible values are 'class' | "
+                    "'metaclass' | 'interface' | 'exception'")
+
+    def __init__(self, name, doc):
+        self.instance_attrs = {}
+        self.locals = {}
+        self.bases = []
+        self.body = []
+        self.name = name
+        self.doc = doc
+
+    def _newstyle_impl(self, context=None):
+        if context is None:
+            context = InferenceContext()
+        if self._newstyle is not None:
+            return self._newstyle
+        for base in self.ancestors(recurs=False, context=context):
+            if base._newstyle_impl(context):
+                self._newstyle = True
+                break
+        if self._newstyle is None:
+            self._newstyle = False
+        return self._newstyle
+
+    _newstyle = None
+    newstyle = property(_newstyle_impl,
+                        doc="boolean indicating if it's a new style class"
+                        "or not")
+
+    def set_line_info(self, lastchild):
+        self.fromlineno = self.lineno
+        self.blockstart_tolineno = self.bases and self.bases[-1].tolineno or self.fromlineno
+        if lastchild is not None:
+            self.tolineno = lastchild.tolineno
+        # else this is a class with only a docstring, then tolineno is (should be) already ok
+
+    def block_range(self, lineno):
+        """return block line numbers.
+
+        start from the "class" position whatever the given lineno
+        """
+        return self.fromlineno, self.tolineno
+
+    def pytype(self):
+        if self.newstyle:
+            return '%s.type' % BUILTINS_MODULE
+        return '%s.classobj' % BUILTINS_MODULE
+
+    def display_type(self):
+        return 'Class'
+
+    def callable(self):
+        return True
+
+    def infer_call_result(self, caller, context=None):
+        """infer what a class is returning when called"""
+        yield Instance(self)
+
+    def scope_lookup(self, node, name, offset=0):
+        if node in self.bases:
+            frame = self.parent.frame()
+            # line offset to avoid that class A(A) resolve the ancestor to
+            # the defined class
+            offset = -1
+        else:
+            frame = self
+        return frame._scope_lookup(node, name, offset)
+
+    # list of parent class as a list of string (i.e. names as they appear
+    # in the class definition) XXX bw compat
+    def basenames(self):
+        return [bnode.as_string() for bnode in self.bases]
+    basenames = property(basenames)
+
+    def ancestors(self, recurs=True, context=None):
+        """return an iterator on the node base classes in a prefixed
+        depth first order
+
+        :param recurs:
+          boolean indicating if it should recurse or return direct
+          ancestors only
+        """
+        # FIXME: should be possible to choose the resolution order
+        # XXX inference make infinite loops possible here (see BaseTransformer
+        # manipulation in the builder module for instance)
+        yielded = set([self])
+        if context is None:
+            context = InferenceContext()
+        for stmt in self.bases:
+            with context.restore_path():
+                try:
+                    for baseobj in stmt.infer(context):
+                        if not isinstance(baseobj, Class):
+                            # duh ?
+                            continue
+                        if baseobj in yielded:
+                            continue # cf xxx above
+                        yielded.add(baseobj)
+                        yield baseobj
+                        if recurs:
+                            for grandpa in baseobj.ancestors(True, context):
+                                if grandpa in yielded:
+                                    continue # cf xxx above
+                                yielded.add(grandpa)
+                                yield grandpa
+                except InferenceError:
+                    # XXX log error ?
+                    continue
+
+    def local_attr_ancestors(self, name, context=None):
+        """return an iterator on astng representation of parent classes
+        which have <name> defined in their locals
+        """
+        for astng in self.ancestors(context=context):
+            if name in astng:
+                yield astng
+
+    def instance_attr_ancestors(self, name, context=None):
+        """return an iterator on astng representation of parent classes
+        which have <name> defined in their instance attribute dictionary
+        """
+        for astng in self.ancestors(context=context):
+            if name in astng.instance_attrs:
+                yield astng
+
+    def has_base(self, node):
+        return node in self.bases
+
+    def local_attr(self, name, context=None):
+        """return the list of assign node associated to name in this class
+        locals or in its parents
+
+        :raises `NotFoundError`:
+          if no attribute with this name has been find in this class or
+          its parent classes
+        """
+        try:
+            return self.locals[name]
+        except KeyError:
+            # get if from the first parent implementing it if any
+            for class_node in self.local_attr_ancestors(name, context):
+                return class_node.locals[name]
+        raise NotFoundError(name)
+    local_attr = remove_nodes(local_attr, DelAttr)
+
+    def instance_attr(self, name, context=None):
+        """return the astng nodes associated to name in this class instance
+        attributes dictionary and in its parents
+
+        :raises `NotFoundError`:
+          if no attribute with this name has been find in this class or
+          its parent classes
+        """
+        values = self.instance_attrs.get(name, [])
+        # get all values from parents
+        for class_node in self.instance_attr_ancestors(name, context):
+            values += class_node.instance_attrs[name]
+        if not values:
+            raise NotFoundError(name)
+        return values
+    instance_attr = remove_nodes(instance_attr, DelAttr)
+
+    def instanciate_class(self):
+        """return Instance of Class node, else return self"""
+        return Instance(self)
+
+    def getattr(self, name, context=None):
+        """this method doesn't look in the instance_attrs dictionary since it's
+        done by an Instance proxy at inference time.
+
+        It may return a YES object if the attribute has not been actually
+        found but a __getattr__ or __getattribute__ method is defined
+        """
+        values = self.locals.get(name, [])
+        if name in self.special_attributes:
+            if name == '__module__':
+                return [cf(self.root().qname())] + values
+            # FIXME : what is expected by passing the list of ancestors to cf:
+            # you can just do [cf(tuple())] + values without breaking any test
+            # this is ticket http://www.logilab.org/ticket/52785
+            if name == '__bases__':
+                return [cf(tuple(self.ancestors(recurs=False, context=context)))] + values
+            # XXX need proper meta class handling + MRO implementation
+            if name == '__mro__' and self.newstyle:
+                # XXX mro is read-only but that's not our job to detect that
+                return [cf(tuple(self.ancestors(recurs=True, context=context)))] + values
+            return std_special_attributes(self, name)
+        # don't modify the list in self.locals!
+        values = list(values)
+        for classnode in self.ancestors(recurs=True, context=context):
+            values += classnode.locals.get(name, [])
+        if not values:
+            raise NotFoundError(name)
+        return values
+
+    def igetattr(self, name, context=None):
+        """inferred getattr, need special treatment in class to handle
+        descriptors
+        """
+        # set lookup name since this is necessary to infer on import nodes for
+        # instance
+        context = copy_context(context)
+        context.lookupname = name
+        try:
+            for infered in _infer_stmts(self.getattr(name, context), context,
+                                        frame=self):
+                # yield YES object instead of descriptors when necessary
+                if not isinstance(infered, Const) and isinstance(infered, Instance):
+                    try:
+                        infered._proxied.getattr('__get__', context)
+                    except NotFoundError:
+                        yield infered
+                    else:
+                        yield YES
+                else:
+                    yield function_to_method(infered, self)
+        except NotFoundError:
+            if not name.startswith('__') and self.has_dynamic_getattr(context):
+                # class handle some dynamic attributes, return a YES object
+                yield YES
+            else:
+                raise InferenceError(name)
+
+    def has_dynamic_getattr(self, context=None):
+        """return True if the class has a custom __getattr__ or
+        __getattribute__ method
+        """
+        # need to explicitly handle optparse.Values (setattr is not detected)
+        if self.name == 'Values' and self.root().name == 'optparse':
+            return True
+        try:
+            self.getattr('__getattr__', context)
+            return True
+        except NotFoundError:
+            #if self.newstyle: XXX cause an infinite recursion error
+            try:
+                getattribute = self.getattr('__getattribute__', context)[0]
+                if getattribute.root().name != BUILTINS_NAME:
+                    # class has a custom __getattribute__ defined
+                    return True
+            except NotFoundError:
+                pass
+        return False
+
+    def methods(self):
+        """return an iterator on all methods defined in the class and
+        its ancestors
+        """
+        done = {}
+        for astng in chain(iter((self,)), self.ancestors()):
+            for meth in astng.mymethods():
+                if meth.name in done:
+                    continue
+                done[meth.name] = None
+                yield meth
+
+    def mymethods(self):
+        """return an iterator on all methods defined in the class"""
+        for member in self.values():
+            if isinstance(member, Function):
+                yield member
+
+    def interfaces(self, herited=True, handler_func=_iface_hdlr):
+        """return an iterator on interfaces implemented by the given
+        class node
+        """
+        # FIXME: what if __implements__ = (MyIFace, MyParent.__implements__)...
+        try:
+            implements = Instance(self).getattr('__implements__')[0]
+        except NotFoundError:
+            return
+        if not herited and not implements.frame() is self:
+            return
+        found = set()
+        missing = False
+        for iface in unpack_infer(implements):
+            if iface is YES:
+                missing = True
+                continue
+            if not iface in found and handler_func(iface):
+                found.add(iface)
+                yield iface
+        if missing:
+            raise InferenceError()

pylibs/logilab/astng/utils.py

@@ -0,0 +1,241 @@
+# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+# copyright 2003-2010 Sylvain Thenault, all rights reserved.
+# contact mailto:thenault@gmail.com
+#
+# This file is part of logilab-astng.
+#
+# logilab-astng is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Lesser General Public License as published by the
+# Free Software Foundation, either version 2.1 of the License, or (at your
+# option) any later version.
+#
+# logilab-astng is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License
+# for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with logilab-astng. If not, see <http://www.gnu.org/licenses/>.
+"""this module contains some utilities to navigate in the tree or to
+extract information from it
+"""
+
+__docformat__ = "restructuredtext en"
+
+from logilab.astng.exceptions import ASTNGBuildingException
+
+
+class ASTWalker:
+    """a walker visiting a tree in preorder, calling on the handler:
+
+    * visit_<class name> on entering a node, where class name is the class of
+    the node in lower case
+
+    * leave_<class name> on leaving a node, where class name is the class of
+    the node in lower case
+    """
+
+    def __init__(self, handler):
+        self.handler = handler
+        self._cache = {}
+
+    def walk(self, node, _done=None):
+        """walk on the tree from <node>, getting callbacks from handler"""
+        if _done is None:
+            _done = set()
+        if node in _done:
+            raise AssertionError((id(node), node, node.parent))
+        _done.add(node)
+        self.visit(node)
+        for child_node in node.get_children():
+            self.handler.set_context(node, child_node)
+            assert child_node is not node
+            self.walk(child_node, _done)
+        self.leave(node)
+        assert node.parent is not node
+
+    def get_callbacks(self, node):
+        """get callbacks from handler for the visited node"""
+        klass = node.__class__
+        methods = self._cache.get(klass)
+        if methods is None:
+            handler = self.handler
+            kid = klass.__name__.lower()
+            e_method = getattr(handler, 'visit_%s' % kid,
+                               getattr(handler, 'visit_default', None))
+            l_method = getattr(handler, 'leave_%s' % kid,
+                               getattr(handler, 'leave_default', None))
+            self._cache[klass] = (e_method, l_method)
+        else:
+            e_method, l_method = methods
+        return e_method, l_method
+
+    def visit(self, node):
+        """walk on the tree from <node>, getting callbacks from handler"""
+        method = self.get_callbacks(node)[0]
+        if method is not None:
+            method(node)
+
+    def leave(self, node):
+        """walk on the tree from <node>, getting callbacks from handler"""
+        method = self.get_callbacks(node)[1]
+        if method is not None:
+            method(node)
+
+
+class LocalsVisitor(ASTWalker):
+    """visit a project by traversing the locals dictionary"""
+    def __init__(self):
+        ASTWalker.__init__(self, self)
+        self._visited = {}
+
+    def visit(self, node):
+        """launch the visit starting from the given node"""
+        if node in self._visited:
+            return
+        self._visited[node] = 1 # FIXME: use set ?
+        methods = self.get_callbacks(node)
+        if methods[0] is not None:
+            methods[0](node)
+        if 'locals' in node.__dict__: # skip Instance and other proxy
+            for name, local_node in node.items():
+                self.visit(local_node)
+        if methods[1] is not None:
+            return methods[1](node)
+
+
+def _check_children(node):
+    """a helper function to check children - parent relations"""
+    for child in node.get_children():
+        ok = False
+        if child is None:
+            print "Hm, child of %s is None" % node
+            continue
+        if not hasattr(child, 'parent'):
+            print " ERROR: %s has child %s %x with no parent" % (node, child, id(child))
+        elif not child.parent:
+            print " ERROR: %s has child %s %x with parent %r" % (node, child, id(child), child.parent)
+        elif child.parent is not node:
+            print " ERROR: %s %x has child %s %x with wrong parent %s" % (node,
+                                      id(node), child, id(child), child.parent)
+        else:
+            ok = True
+        if not ok:
+            print "lines;", node.lineno, child.lineno
+            print "of module", node.root(), node.root().name
+            raise ASTNGBuildingException
+        _check_children(child)
+
+
+from _ast import PyCF_ONLY_AST
+def parse(string):
+    return compile(string, "<string>", 'exec', PyCF_ONLY_AST)
+
+class TreeTester(object):
+    '''A helper class to see _ast tree and compare with astng tree
+
+    indent: string for tree indent representation
+    lineno: bool to tell if we should print the line numbers
+
+    >>> tester = TreeTester('print')
+    >>> print tester.native_tree_repr()
+
+    <Module>
+    .   body = [
+    .   <Print>
+    .   .   nl = True
+    .   ]
+    >>> print tester.astng_tree_repr()
+    Module()
+        body = [
+        Print()
+            dest = 
+            values = [
+            ]
+        ]
+    '''
+
+    indent = '.   '
+    lineno = False
+
+    def __init__(self, sourcecode):
+        self._string = ''
+        self.sourcecode = sourcecode
+        self._ast_node = None
+        self.build_ast()
+
+    def build_ast(self):
+        """build the _ast tree from the source code"""
+        self._ast_node = parse(self.sourcecode)
+
+    def native_tree_repr(self, node=None, indent=''):
+        """get a nice representation of the _ast tree"""
+        self._string = ''
+        if node is None:
+            node = self._ast_node
+        self._native_repr_tree(node, indent)
+        return self._string
+
+
+    def _native_repr_tree(self, node, indent, _done=None):
+        """recursive method for the native tree representation"""
+        from _ast import Load as _Load, Store as _Store, Del as _Del
+        from _ast import AST as Node
+        if _done is None:
+            _done = set()
+        if node in _done:
+            self._string += '\nloop in tree: %r (%s)' % (node,
+                                            getattr(node, 'lineno', None))
+            return
+        _done.add(node)
+        self._string += '\n' + indent +  '<%s>' % node.__class__.__name__
+        indent += self.indent
+        if not hasattr(node, '__dict__'):
+            self._string += '\n' + self.indent + " ** node has no __dict__ " + str(node)
+            return
+        node_dict = node.__dict__
+        if hasattr(node, '_attributes'):
+            for a in node._attributes:
+                attr = node_dict[a]
+                if attr is None:
+                    continue
+                if a in ("lineno", "col_offset") and not self.lineno:
+                    continue
+                self._string +='\n' +  indent + a + " = " + repr(attr)
+        for field in node._fields or ():
+            attr = node_dict[field]
+            if attr is None:
+                continue
+            if isinstance(attr, list):
+                if not attr:
+                    continue
+                self._string += '\n' + indent + field + ' = ['
+                for elt in attr:
+                    self._native_repr_tree(elt, indent, _done)
+                self._string += '\n' + indent + ']'
+                continue
+            if isinstance(attr, (_Load, _Store, _Del)):
+                continue
+            if isinstance(attr, Node):
+                self._string += '\n' + indent + field + " = "
+                self._native_repr_tree(attr, indent, _done)
+            else:
+                self._string += '\n' + indent + field + " = " + repr(attr)
+
+
+    def build_astng_tree(self):
+        """build astng tree from the _ast tree
+        """
+        from logilab.astng.builder import ASTNGBuilder
+        tree = ASTNGBuilder().string_build(self.sourcecode)
+        return tree
+
+    def astng_tree_repr(self, ids=False):
+        """build the astng tree and return a nice tree representation"""
+        mod = self.build_astng_tree()
+        return mod.repr_tree(ids)
+
+
+__all__ = ('LocalsVisitor', 'ASTWalker',)
+

pylibs/logilab/common/COPYING

@@ -0,0 +1,339 @@
+		    GNU GENERAL PUBLIC LICENSE
+		       Version 2, June 1991
+
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+			    Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+License is intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users.  This
+General Public License applies to most of the Free Software
+Foundation's software and to any other program whose authors commit to
+using it.  (Some other Free Software Foundation software is covered by
+the GNU Lesser General Public License instead.)  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+this service if you wish), that you receive source code or can get it
+if you want it, that you can change the software or use pieces of it
+in new free programs; and that you know you can do these things.
+
+  To protect your rights, we need to make restrictions that forbid
+anyone to deny you these rights or to ask you to surrender the rights.
+These restrictions translate to certain responsibilities for you if you
+distribute copies of the software, or if you modify it.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must give the recipients all the rights that
+you have.  You must make sure that they, too, receive or can get the
+source code.  And you must show them these terms so they know their
+rights.
+
+  We protect your rights with two steps: (1) copyright the software, and
+(2) offer you this license which gives you legal permission to copy,
+distribute and/or modify the software.
+
+  Also, for each author's protection and ours, we want to make certain
+that everyone understands that there is no warranty for this free
+software.  If the software is modified by someone else and passed on, we
+want its recipients to know that what they have is not the original, so
+that any problems introduced by others will not reflect on the original
+authors' reputations.
+
+  Finally, any free program is threatened constantly by software
+patents.  We wish to avoid the danger that redistributors of a free
+program will individually obtain patent licenses, in effect making the
+program proprietary.  To prevent this, we have made it clear that any
+patent must be licensed for everyone's free use or not licensed at all.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+		    GNU GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License.  The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language.  (Hereinafter, translation is included without limitation in
+the term "modification".)  Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+  1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+  2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) You must cause the modified files to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    b) You must cause any work that you distribute or publish, that in
+    whole or in part contains or is derived from the Program or any
+    part thereof, to be licensed as a whole at no charge to all third
+    parties under the terms of this License.
+
+    c) If the modified program normally reads commands interactively
+    when run, you must cause it, when started running for such
+    interactive use in the most ordinary way, to print or display an
+    announcement including an appropriate copyright notice and a
+    notice that there is no warranty (or else, saying that you provide
+    a warranty) and that users may redistribute the program under
+    these conditions, and telling the user how to view a copy of this
+    License.  (Exception: if the Program itself is interactive but
+    does not normally print such an announcement, your work based on
+    the Program is not required to print an announcement.)
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+    a) Accompany it with the complete corresponding machine-readable
+    source code, which must be distributed under the terms of Sections
+    1 and 2 above on a medium customarily used for software interchange; or,
+
+    b) Accompany it with a written offer, valid for at least three
+    years, to give any third party, for a charge no more than your
+    cost of physically performing source distribution, a complete
+    machine-readable copy of the corresponding source code, to be
+    distributed under the terms of Sections 1 and 2 above on a medium
+    customarily used for software interchange; or,
+
+    c) Accompany it with the information you received as to the offer
+    to distribute corresponding source code.  (This alternative is
+    allowed only for noncommercial distribution and only if you
+    received the program in object code or executable form with such
+    an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it.  For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable.  However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+  4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License.  Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+  5. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Program or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+  6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+  7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+  8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation.  If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+  10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission.  For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this.  Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+			    NO WARRANTY
+
+  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+		     END OF TERMS AND CONDITIONS
+
+	    How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+Also add information on how to contact you by electronic and paper mail.
+
+If the program is interactive, make it output a short notice like this
+when it starts in an interactive mode:
+
+    Gnomovision version 69, Copyright (C) year name of author
+    Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, the commands you use may
+be called something other than `show w' and `show c'; they could even be
+mouse-clicks or menu items--whatever suits your program.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the program, if
+necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the program
+  `Gnomovision' (which makes passes at compilers) written by James Hacker.
+
+  <signature of Ty Coon>, 1 April 1989
+  Ty Coon, President of Vice
+
+This General Public License does not permit incorporating your program into
+proprietary programs.  If your program is a subroutine library, you may
+consider it more useful to permit linking proprietary applications with the
+library.  If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.

pylibs/logilab/common/COPYING.LESSER

@@ -0,0 +1,510 @@
+
+                  GNU LESSER GENERAL PUBLIC LICENSE
+                       Version 2.1, February 1999
+
+ Copyright (C) 1991, 1999 Free Software Foundation, Inc.
+	51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+[This is the first released version of the Lesser GPL.  It also counts
+ as the successor of the GNU Library Public License, version 2, hence
+ the version number 2.1.]
+
+                            Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+Licenses are intended to guarantee your freedom to share and change
+free software--to make sure the software is free for all its users.
+
+  This license, the Lesser General Public License, applies to some
+specially designated software packages--typically libraries--of the
+Free Software Foundation and other authors who decide to use it.  You
+can use it too, but we suggest you first think carefully about whether
+this license or the ordinary General Public License is the better
+strategy to use in any particular case, based on the explanations
+below.
+
+  When we speak of free software, we are referring to freedom of use,
+not price.  Our General Public Licenses are designed to make sure that
+you have the freedom to distribute copies of free software (and charge
+for this service if you wish); that you receive source code or can get
+it if you want it; that you can change the software and use pieces of
+it in new free programs; and that you are informed that you can do
+these things.
+
+  To protect your rights, we need to make restrictions that forbid
+distributors to deny you these rights or to ask you to surrender these
+rights.  These restrictions translate to certain responsibilities for
+you if you distribute copies of the library or if you modify it.
+
+  For example, if you distribute copies of the library, whether gratis
+or for a fee, you must give the recipients all the rights that we gave
+you.  You must make sure that they, too, receive or can get the source
+code.  If you link other code with the library, you must provide
+complete object files to the recipients, so that they can relink them
+with the library after making changes to the library and recompiling
+it.  And you must show them these terms so they know their rights.
+
+  We protect your rights with a two-step method: (1) we copyright the
+library, and (2) we offer you this license, which gives you legal
+permission to copy, distribute and/or modify the library.
+
+  To protect each distributor, we want to make it very clear that
+there is no warranty for the free library.  Also, if the library is
+modified by someone else and passed on, the recipients should know
+that what they have is not the original version, so that the original
+author's reputation will not be affected by problems that might be
+introduced by others.
+
+  Finally, software patents pose a constant threat to the existence of
+any free program.  We wish to make sure that a company cannot
+effectively restrict the users of a free program by obtaining a
+restrictive license from a patent holder.  Therefore, we insist that
+any patent license obtained for a version of the library must be
+consistent with the full freedom of use specified in this license.
+
+  Most GNU software, including some libraries, is covered by the
+ordinary GNU General Public License.  This license, the GNU Lesser
+General Public License, applies to certain designated libraries, and
+is quite different from the ordinary General Public License.  We use
+this license for certain libraries in order to permit linking those
+libraries into non-free programs.
+
+  When a program is linked with a library, whether statically or using
+a shared library, the combination of the two is legally speaking a
+combined work, a derivative of the original library.  The ordinary
+General Public License therefore permits such linking only if the
+entire combination fits its criteria of freedom.  The Lesser General
+Public License permits more lax criteria for linking other code with
+the library.
+
+  We call this license the "Lesser" General Public License because it
+does Less to protect the user's freedom than the ordinary General
+Public License.  It also provides other free software developers Less
+of an advantage over competing non-free programs.  These disadvantages
+are the reason we use the ordinary General Public License for many
+libraries.  However, the Lesser license provides advantages in certain
+special circumstances.
+
+  For example, on rare occasions, there may be a special need to
+encourage the widest possible use of a certain library, so that it
+becomes a de-facto standard.  To achieve this, non-free programs must
+be allowed to use the library.  A more frequent case is that a free
+library does the same job as widely used non-free libraries.  In this
+case, there is little to gain by limiting the free library to free
+software only, so we use the Lesser General Public License.
+
+  In other cases, permission to use a particular library in non-free
+programs enables a greater number of people to use a large body of
+free software.  For example, permission to use the GNU C Library in
+non-free programs enables many more people to use the whole GNU
+operating system, as well as its variant, the GNU/Linux operating
+system.
+
+  Although the Lesser General Public License is Less protective of the
+users' freedom, it does ensure that the user of a program that is
+linked with the Library has the freedom and the wherewithal to run
+that program using a modified version of the Library.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.  Pay close attention to the difference between a
+"work based on the library" and a "work that uses the library".  The
+former contains code derived from the library, whereas the latter must
+be combined with the library in order to run.
+
+                  GNU LESSER GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License Agreement applies to any software library or other
+program which contains a notice placed by the copyright holder or
+other authorized party saying it may be distributed under the terms of
+this Lesser General Public License (also called "this License").
+Each licensee is addressed as "you".
+
+  A "library" means a collection of software functions and/or data
+prepared so as to be conveniently linked with application programs
+(which use some of those functions and data) to form executables.
+
+  The "Library", below, refers to any such software library or work
+which has been distributed under these terms.  A "work based on the
+Library" means either the Library or any derivative work under
+copyright law: that is to say, a work containing the Library or a
+portion of it, either verbatim or with modifications and/or translated
+straightforwardly into another language.  (Hereinafter, translation is
+included without limitation in the term "modification".)
+
+  "Source code" for a work means the preferred form of the work for
+making modifications to it.  For a library, complete source code means
+all the source code for all modules it contains, plus any associated
+interface definition files, plus the scripts used to control
+compilation and installation of the library.
+
+  Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running a program using the Library is not restricted, and output from
+such a program is covered only if its contents constitute a work based
+on the Library (independent of the use of the Library in a tool for
+writing it).  Whether that is true depends on what the Library does
+and what the program that uses the Library does.
+
+  1. You may copy and distribute verbatim copies of the Library's
+complete source code as you receive it, in any medium, provided that
+you conspicuously and appropriately publish on each copy an
+appropriate copyright notice and disclaimer of warranty; keep intact
+all the notices that refer to this License and to the absence of any
+warranty; and distribute a copy of this License along with the
+Library.
+
+  You may charge a fee for the physical act of transferring a copy,
+and you may at your option offer warranty protection in exchange for a
+fee.
+
+  2. You may modify your copy or copies of the Library or any portion
+of it, thus forming a work based on the Library, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) The modified work must itself be a software library.
+
+    b) You must cause the files modified to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    c) You must cause the whole of the work to be licensed at no
+    charge to all third parties under the terms of this License.
+
+    d) If a facility in the modified Library refers to a function or a
+    table of data to be supplied by an application program that uses
+    the facility, other than as an argument passed when the facility
+    is invoked, then you must make a good faith effort to ensure that,
+    in the event an application does not supply such function or
+    table, the facility still operates, and performs whatever part of
+    its purpose remains meaningful.
+
+    (For example, a function in a library to compute square roots has
+    a purpose that is entirely well-defined independent of the
+    application.  Therefore, Subsection 2d requires that any
+    application-supplied function or table used by this function must
+    be optional: if the application does not supply it, the square
+    root function must still compute square roots.)
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Library,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Library, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote
+it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Library.
+
+In addition, mere aggregation of another work not based on the Library
+with the Library (or with a work based on the Library) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may opt to apply the terms of the ordinary GNU General Public
+License instead of this License to a given copy of the Library.  To do
+this, you must alter all the notices that refer to this License, so
+that they refer to the ordinary GNU General Public License, version 2,
+instead of to this License.  (If a newer version than version 2 of the
+ordinary GNU General Public License has appeared, then you can specify
+that version instead if you wish.)  Do not make any other change in
+these notices.
+
+  Once this change is made in a given copy, it is irreversible for
+that copy, so the ordinary GNU General Public License applies to all
+subsequent copies and derivative works made from that copy.
+
+  This option is useful when you wish to copy part of the code of
+the Library into a program that is not a library.
+
+  4. You may copy and distribute the Library (or a portion or
+derivative of it, under Section 2) in object code or executable form
+under the terms of Sections 1 and 2 above provided that you accompany
+it with the complete corresponding machine-readable source code, which
+must be distributed under the terms of Sections 1 and 2 above on a
+medium customarily used for software interchange.
+
+  If distribution of object code is made by offering access to copy
+from a designated place, then offering equivalent access to copy the
+source code from the same place satisfies the requirement to
+distribute the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+  5. A program that contains no derivative of any portion of the
+Library, but is designed to work with the Library by being compiled or
+linked with it, is called a "work that uses the Library".  Such a
+work, in isolation, is not a derivative work of the Library, and
+therefore falls outside the scope of this License.
+
+  However, linking a "work that uses the Library" with the Library
+creates an executable that is a derivative of the Library (because it
+contains portions of the Library), rather than a "work that uses the
+library".  The executable is therefore covered by this License.
+Section 6 states terms for distribution of such executables.
+
+  When a "work that uses the Library" uses material from a header file
+that is part of the Library, the object code for the work may be a
+derivative work of the Library even though the source code is not.
+Whether this is true is especially significant if the work can be
+linked without the Library, or if the work is itself a library.  The
+threshold for this to be true is not precisely defined by law.
+
+  If such an object file uses only numerical parameters, data
+structure layouts and accessors, and small macros and small inline
+functions (ten lines or less in length), then the use of the object
+file is unrestricted, regardless of whether it is legally a derivative
+work.  (Executables containing this object code plus portions of the
+Library will still fall under Section 6.)
+
+  Otherwise, if the work is a derivative of the Library, you may
+distribute the object code for the work under the terms of Section 6.
+Any executables containing that work also fall under Section 6,
+whether or not they are linked directly with the Library itself.
+
+  6. As an exception to the Sections above, you may also combine or
+link a "work that uses the Library" with the Library to produce a
+work containing portions of the Library, and distribute that work
+under terms of your choice, provided that the terms permit
+modification of the work for the customer's own use and reverse
+engineering for debugging such modifications.
+
+  You must give prominent notice with each copy of the work that the
+Library is used in it and that the Library and its use are covered by
+this License.  You must supply a copy of this License.  If the work
+during execution displays copyright notices, you must include the
+copyright notice for the Library among them, as well as a reference
+directing the user to the copy of this License.  Also, you must do one
+of these things:
+
+    a) Accompany the work with the complete corresponding
+    machine-readable source code for the Library including whatever
+    changes were used in the work (which must be distributed under
+    Sections 1 and 2 above); and, if the work is an executable linked
+    with the Library, with the complete machine-readable "work that
+    uses the Library", as object code and/or source code, so that the
+    user can modify the Library and then relink to produce a modified
+    executable containing the modified Library.  (It is understood
+    that the user who changes the contents of definitions files in the
+    Library will not necessarily be able to recompile the application
+    to use the modified definitions.)
+
+    b) Use a suitable shared library mechanism for linking with the
+    Library.  A suitable mechanism is one that (1) uses at run time a
+    copy of the library already present on the user's computer system,
+    rather than copying library functions into the executable, and (2)
+    will operate properly with a modified version of the library, if
+    the user installs one, as long as the modified version is
+    interface-compatible with the version that the work was made with.
+
+    c) Accompany the work with a written offer, valid for at least
+    three years, to give the same user the materials specified in
+    Subsection 6a, above, for a charge no more than the cost of
+    performing this distribution.
+
+    d) If distribution of the work is made by offering access to copy
+    from a designated place, offer equivalent access to copy the above
+    specified materials from the same place.
+
+    e) Verify that the user has already received a copy of these
+    materials or that you have already sent this user a copy.
+
+  For an executable, the required form of the "work that uses the
+Library" must include any data and utility programs needed for
+reproducing the executable from it.  However, as a special exception,
+the materials to be distributed need not include anything that is
+normally distributed (in either source or binary form) with the major
+components (compiler, kernel, and so on) of the operating system on
+which the executable runs, unless that component itself accompanies
+the executable.
+
+  It may happen that this requirement contradicts the license
+restrictions of other proprietary libraries that do not normally
+accompany the operating system.  Such a contradiction means you cannot
+use both them and the Library together in an executable that you
+distribute.
+
+  7. You may place library facilities that are a work based on the
+Library side-by-side in a single library together with other library
+facilities not covered by this License, and distribute such a combined
+library, provided that the separate distribution of the work based on
+the Library and of the other library facilities is otherwise
+permitted, and provided that you do these two things:
+
+    a) Accompany the combined library with a copy of the same work
+    based on the Library, uncombined with any other library
+    facilities.  This must be distributed under the terms of the
+    Sections above.
+
+    b) Give prominent notice with the combined library of the fact
+    that part of it is a work based on the Library, and explaining
+    where to find the accompanying uncombined form of the same work.
+
+  8. You may not copy, modify, sublicense, link with, or distribute
+the Library except as expressly provided under this License.  Any
+attempt otherwise to copy, modify, sublicense, link with, or
+distribute the Library is void, and will automatically terminate your
+rights under this License.  However, parties who have received copies,
+or rights, from you under this License will not have their licenses
+terminated so long as such parties remain in full compliance.
+
+  9. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Library or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Library (or any work based on the
+Library), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Library or works based on it.
+
+  10. Each time you redistribute the Library (or any work based on the
+Library), the recipient automatically receives a license from the
+original licensor to copy, distribute, link with or modify the Library
+subject to these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties with
+this License.
+
+  11. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Library at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Library by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Library.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply, and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+  12. If the distribution and/or use of the Library is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Library under this License
+may add an explicit geographical distribution limitation excluding those
+countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  13. The Free Software Foundation may publish revised and/or new
+versions of the Lesser General Public License from time to time.
+Such new versions will be similar in spirit to the present version,
+but may differ in detail to address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Library
+specifies a version number of this License which applies to it and
+"any later version", you have the option of following the terms and
+conditions either of that version or of any later version published by
+the Free Software Foundation.  If the Library does not specify a
+license version number, you may choose any version ever published by
+the Free Software Foundation.
+
+  14. If you wish to incorporate parts of the Library into other free
+programs whose distribution conditions are incompatible with these,
+write to the author to ask for permission.  For software which is
+copyrighted by the Free Software Foundation, write to the Free
+Software Foundation; we sometimes make exceptions for this.  Our
+decision will be guided by the two goals of preserving the free status
+of all derivatives of our free software and of promoting the sharing
+and reuse of software generally.
+
+                            NO WARRANTY
+
+  15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO
+WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW.
+EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR
+OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY
+KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE
+LIBRARY IS WITH YOU.  SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME
+THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN
+WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY
+AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU
+FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR
+CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE
+LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING
+RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A
+FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF
+SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
+DAMAGES.
+
+                     END OF TERMS AND CONDITIONS
+
+           How to Apply These Terms to Your New Libraries
+
+  If you develop a new library, and you want it to be of the greatest
+possible use to the public, we recommend making it free software that
+everyone can redistribute and change.  You can do so by permitting
+redistribution under these terms (or, alternatively, under the terms
+of the ordinary General Public License).
+
+  To apply these terms, attach the following notices to the library.
+It is safest to attach them to the start of each source file to most
+effectively convey the exclusion of warranty; and each file should
+have at least the "copyright" line and a pointer to where the full
+notice is found.
+
+
+    <one line to give the library's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation; either
+    version 2.1 of the License, or (at your option) any later version.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+Also add information on how to contact you by electronic and paper mail.
+
+You should also get your employer (if you work as a programmer) or
+your school, if any, to sign a "copyright disclaimer" for the library,
+if necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the
+  library `Frob' (a library for tweaking knobs) written by James
+  Random Hacker.
+
+  <signature of Ty Coon>, 1 April 1990
+  Ty Coon, President of Vice
+
+That's all there is to it!
+
+

pylibs/logilab/common/README

@@ -0,0 +1,187 @@
+Logilab's common library
+========================
+
+What's this ?
+-------------
+
+This package contains some modules used by differents Logilab's projects.
+
+It is released under the GNU Lesser General Public License.
+
+There is no documentation available yet but the source code should be clean and
+well documented.
+
+Designed to ease:
+
+* handling command line options and configuration files
+* writing interactive command line tools
+* manipulation of files and character strings
+* manipulation of common structures such as graph, tree, and pattern such as visitor
+* generating text and HTML reports
+* accessing some external libraries such as OmniORB_, Pyro_...
+* more...
+
+
+Installation
+------------
+
+Extract the tarball, jump into the created directory and run ::
+
+	python setup.py install
+
+For installation options, see ::
+
+	python setup.py install --help
+
+
+Provided modules
+----------------
+
+Here is a brief description of the available modules.
+
+Modules providing high-level features
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* `cache`, a cache implementation with a least recently used algorithm.
+
+* `changelog`, a tiny library to manipulate our simplified ChangeLog file format.
+
+* `clcommands`, high-level classes to define command line programs handling
+  different subcommands. It is based on `configuration` to get easy command line
+  / configuration file handling.
+
+* `cli`, a base class for interactive programs using the command line.
+
+* `configuration`, some classes to handle unified configuration from both
+  command line (using optparse) and configuration file (using ConfigParser).
+
+* `dbf`, read Visual Fox Pro DBF files.
+
+* `proc`, interface to Linux /proc.
+
+* `umessage`, unicode email support.
+
+* `ureports`, micro-reports, a way to create simple reports using python objects
+  without care of the final formatting. ReST and html formatters are provided.
+
+
+Modules providing low-level functions and structures
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* `compat`, provides a transparent compatibility layer between different python
+  versions.
+
+* `date`, a set of date manipulation functions.
+
+* `daemon`, a daemon function and mix-in class to properly start an Unix daemon
+  process.
+
+* `decorators`, function decorators such as cached, timed...
+
+* `deprecation`, decorator, metaclass & all to mark functions / classes as
+  deprecated or moved
+
+* `fileutils`, some file / file path manipulation utilities.
+
+* `graph`, graph manipulations functions such as cycle detection, bases for dot
+  file generation.
+
+* `modutils`, python module manipulation functions.
+
+* `shellutils`, some powerful shell like functions to replace shell scripts with
+  python scripts.
+
+* `tasksqueue`, a prioritized tasks queue implementation.
+
+* `textutils`, some text manipulation functions (ansi colorization, line wrapping,
+  rest support...).
+
+* `tree`, base class to represent tree structure, and some others to make it
+  works with the visitor implementation (see below).
+
+* `visitor`, a generic visitor pattern implementation.
+
+
+Modules extending some standard modules
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* `debugger`,  `pdb` customization.
+
+* `logging_ext`, extensions to `logging` module such as a colorized formatter
+  and an easier initialization function.
+
+* `optik_ext`, defines some new option types (regexp, csv, color, date, etc.)
+  for `optik` / `optparse`
+
+* `xmlrpcutils`, auth support for XML-RPC
+
+
+Modules extending some external modules
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* `corbautils`, useful functions for use with the OmniORB_ CORBA library.
+
+* `hg`, some Mercurial_ utility functions.
+
+* `pdf_ext`, pdf and fdf file manipulations, with pdftk.
+
+* `pyro_ext`, some Pyro_ utility functions.
+
+* `sphinx_ext`, Sphinx_ plugin defining a `autodocstring` directive.
+
+* `vcgutils` , utilities functions to generate file readable with Georg Sander's
+  vcg tool (Visualization of Compiler Graphs).
+
+
+To be deprecated modules
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+Those `logilab.common` modules will much probably be deprecated in future
+versions:
+
+* `testlib`: use `unittest2`_ instead
+* `pytest`: use `discover`_ instead
+* `interface`: use `zope.interface`_ if you really want this
+* `table`, `xmlutils`: is that used?
+* `sphinxutils`: we won't go that way imo (i == syt)
+
+
+Deprecated modules
+~~~~~~~~~~~~~~~~~~
+
+Those `logilab.common` modules are only there for backward compatibility. They
+can go away at anytime.
+
+* `optparser`: use `clcommands` instead
+
+* `adbh`, `db`, `sqlgen`: see `logilab.database`_ instead
+
+* `contexts`: content move to `shellutils`
+
+* `html`: deprecated without replacement
+
+
+Comments, support, bug reports
+------------------------------
+
+Project page http://www.logilab.org/project/logilab-common
+
+Use the python-projects@lists.logilab.org mailing list. Since we do not have
+publicly available bug tracker yet, bug reports should be emailed
+there too.
+
+You can subscribe to this mailing list at
+http://lists.logilab.org/mailman/listinfo/python-projects
+
+Archives are available at
+http://lists.logilab.org/pipermail/python-projects/
+
+
+.. _Pyro: http://pyro.sourceforge.net/
+.. _OmniORB: http://omniorb.sourceforge.net/
+.. _Mercurial: http://mercurial.selenic.com
+.. _Sphinx: http://sphinx.pocoo.org/
+.. _`logilab.database`: http://www.logilab.org/project/logilab-database/
+.. _`unittest2`: http://pypi.python.org/pypi/unittest2
+.. _`discover`: http://pypi.python.org/pypi/discover
+.. _`zope.interface`: http://pypi.python.org/pypi/zope.interface

pylibs/logilab/common/README.Python3

@@ -0,0 +1,29 @@
+Python3
+=======
+
+Approach
+--------
+
+We maintain a Python 2 base and use 2to3 to generate Python 3 code.
+
+2to3 is integrated into the distutils installation process and will be run as a
+build step when invoked by the python3 interpreter::
+
+  python3 setup.py install
+
+Tests
+-----
+
+Set your PYTHONPATH and run pytest3 against the test directory.
+
+Debian
+------
+
+For the Debian packaging of python3-logilab-common, you can use the debian.sid/
+content against the debian/ folder::
+
+  cp debian.sid/* debian/
+
+Resources
+---------
+http://wiki.python.org/moin/PortingPythonToPy3k

pylibs/logilab/common/__init__.py

@@ -0,0 +1,171 @@
+# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+#
+# This file is part of logilab-common.
+#
+# logilab-common is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation, either version 2.1 of the License, or (at your option) any
+# later version.
+#
+# logilab-common is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with logilab-common.  If not, see <http://www.gnu.org/licenses/>.
+"""Logilab common library (aka Logilab's extension to the standard library).
+
+:type STD_BLACKLIST: tuple
+:var STD_BLACKLIST: directories ignored by default by the functions in
+  this package which have to recurse into directories
+
+:type IGNORED_EXTENSIONS: tuple
+:var IGNORED_EXTENSIONS: file extensions that may usually be ignored
+"""
+__docformat__ = "restructuredtext en"
+from logilab.common.__pkginfo__ import version as __version__
+
+STD_BLACKLIST = ('CVS', '.svn', '.hg', 'debian', 'dist', 'build')
+
+IGNORED_EXTENSIONS = ('.pyc', '.pyo', '.elc', '~', '.swp', '.orig')
+
+# set this to False if you've mx DateTime installed but you don't want your db
+# adapter to use it (should be set before you got a connection)
+USE_MX_DATETIME = True
+
+
+class attrdict(dict):
+    """A dictionary for which keys are also accessible as attributes."""
+    def __getattr__(self, attr):
+        try:
+            return self[attr]
+        except KeyError:
+            raise AttributeError(attr)
+
+class dictattr(dict):
+    def __init__(self, proxy):
+        self.__proxy = proxy
+
+    def __getitem__(self, attr):
+        try:
+            return getattr(self.__proxy, attr)
+        except AttributeError:
+            raise KeyError(attr)
+
+class nullobject(object):
+    def __repr__(self):
+        return '<nullobject>'
+    def __nonzero__(self):
+        return False
+
+class tempattr(object):
+    def __init__(self, obj, attr, value):
+        self.obj = obj
+        self.attr = attr
+        self.value = value
+
+    def __enter__(self):
+        self.oldvalue = getattr(self.obj, self.attr)
+        setattr(self.obj, self.attr, self.value)
+        return self.obj
+
+    def __exit__(self, exctype, value, traceback):
+        setattr(self.obj, self.attr, self.oldvalue)
+
+
+
+# flatten -----
+# XXX move in a specific module and use yield instead
+# do not mix flatten and translate
+#
+# def iterable(obj):
+#    try: iter(obj)
+#    except: return False
+#    return True
+#
+# def is_string_like(obj):
+#    try: obj +''
+#    except (TypeError, ValueError): return False
+#    return True
+#
+#def is_scalar(obj):
+#    return is_string_like(obj) or not iterable(obj)
+#
+#def flatten(seq):
+#    for item in seq:
+#        if is_scalar(item):
+#            yield item
+#        else:
+#            for subitem in flatten(item):
+#               yield subitem
+
+def flatten(iterable, tr_func=None, results=None):
+    """Flatten a list of list with any level.
+
+    If tr_func is not None, it should be a one argument function that'll be called
+    on each final element.
+
+    :rtype: list
+
+    >>> flatten([1, [2, 3]])
+    [1, 2, 3]
+    """
+    if results is None:
+        results = []
+    for val in iterable:
+        if isinstance(val, (list, tuple)):
+            flatten(val, tr_func, results)
+        elif tr_func is None:
+            results.append(val)
+        else:
+            results.append(tr_func(val))
+    return results
+
+
+# XXX is function below still used ?
+
+def make_domains(lists):
+    """
+    Given a list of lists, return a list of domain for each list to produce all
+    combinations of possibles values.
+
+    :rtype: list
+
+    Example:
+
+    >>> make_domains(['a', 'b'], ['c','d', 'e'])
+    [['a', 'b', 'a', 'b', 'a', 'b'], ['c', 'c', 'd', 'd', 'e', 'e']]
+    """
+    domains = []
+    for iterable in lists:
+        new_domain = iterable[:]
+        for i in range(len(domains)):
+            domains[i] = domains[i]*len(iterable)
+        if domains:
+            missing = (len(domains[0]) - len(iterable)) / len(iterable)
+            i = 0
+            for j in range(len(iterable)):
+                value = iterable[j]
+                for dummy in range(missing):
+                    new_domain.insert(i, value)
+                    i += 1
+                i += 1
+        domains.append(new_domain)
+    return domains
+
+
+# private stuff ################################################################
+
+def _handle_blacklist(blacklist, dirnames, filenames):
+    """remove files/directories in the black list
+
+    dirnames/filenames are usually from os.walk
+    """
+    for norecurs in blacklist:
+        if norecurs in dirnames:
+            dirnames.remove(norecurs)
+        elif norecurs in filenames:
+            filenames.remove(norecurs)
+

pylibs/logilab/common/__pkginfo__.py

@@ -0,0 +1,50 @@
+# copyright 2003-2012 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+#
+# This file is part of logilab-common.
+#
+# logilab-common is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation, either version 2.1 of the License, or (at your option) any
+# later version.
+#
+# logilab-common is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with logilab-common.  If not, see <http://www.gnu.org/licenses/>.
+"""logilab.common packaging information"""
+__docformat__ = "restructuredtext en"
+import sys
+
+distname = 'logilab-common'
+modname = 'common'
+subpackage_of = 'logilab'
+subpackage_master = True
+
+numversion = (0, 58, 0)
+version = '.'.join([str(num) for num in numversion])
+
+license = 'LGPL' # 2.1 or later
+description = "collection of low-level Python packages and modules used by Logilab projects"
+web = "http://www.logilab.org/project/%s" % distname
+ftp = "ftp://ftp.logilab.org/pub/%s" % modname
+mailinglist = "mailto://python-projects@lists.logilab.org"
+author = "Logilab"
+author_email = "contact@logilab.fr"
+
+
+from os.path import join
+scripts = [join('bin', 'pytest')]
+include_dirs = [join('test', 'data')]
+
+if sys.version_info < (2, 7):
+    install_requires = ['unittest2 >= 0.5.1']
+
+classifiers = ["Topic :: Utilities",
+               "Programming Language :: Python",
+               "Programming Language :: Python :: 2",
+               "Programming Language :: Python :: 3",
+               ]

pylibs/logilab/common/announce.txt

@@ -0,0 +1,25 @@
+I'm pleased to announce the %(VERSION)s release of %(DISTNAME)s.
+
+What's new ?
+------------
+%(CHANGELOG)s
+
+
+What is %(DISTNAME)s ?
+------------------------
+%(LONG_DESC)s
+
+
+Home page
+---------
+%(WEB)s
+
+Download
+--------
+%(FTP)s
+
+Mailing list
+------------
+%(MAILINGLIST)s
+
+%(ADDITIONAL_DESCR)s

pylibs/logilab/common/cache.py

@@ -0,0 +1,114 @@
+# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+#
+# This file is part of logilab-common.
+#
+# logilab-common is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation, either version 2.1 of the License, or (at your option) any
+# later version.
+#
+# logilab-common is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with logilab-common.  If not, see <http://www.gnu.org/licenses/>.
+"""Cache module, with a least recently used algorithm for the management of the
+deletion of entries.
+
+
+
+
+"""
+__docformat__ = "restructuredtext en"
+
+from threading import Lock
+
+from logilab.common.decorators import locked
+
+_marker = object()
+
+class Cache(dict):
+    """A dictionary like cache.
+
+    inv:
+        len(self._usage) <= self.size
+        len(self.data) <= self.size
+    """
+
+    def __init__(self, size=100):
+        """ Warning : Cache.__init__() != dict.__init__().
+        Constructor does not take any arguments beside size.
+        """
+        assert size >= 0, 'cache size must be >= 0 (0 meaning no caching)'
+        self.size = size
+        self._usage = []
+        self._lock = Lock()
+        super(Cache, self).__init__()
+
+    def _acquire(self):
+        self._lock.acquire()
+
+    def _release(self):
+        self._lock.release()
+
+    def _update_usage(self, key):
+        if not self._usage:
+            self._usage.append(key)
+        elif self._usage[-1] != key:
+            try:
+                self._usage.remove(key)
+            except ValueError:
+                # we are inserting a new key
+                # check the size of the dictionary
+                # and remove the oldest item in the cache
+                if self.size and len(self._usage) >= self.size:
+                    super(Cache, self).__delitem__(self._usage[0])
+                    del self._usage[0]
+            self._usage.append(key)
+        else:
+            pass # key is already the most recently used key
+
+    def __getitem__(self, key):
+        value = super(Cache, self).__getitem__(key)
+        self._update_usage(key)
+        return value
+    __getitem__ = locked(_acquire, _release)(__getitem__)
+
+    def __setitem__(self, key, item):
+        # Just make sure that size > 0 before inserting a new item in the cache
+        if self.size > 0:
+            super(Cache, self).__setitem__(key, item)
+            self._update_usage(key)
+    __setitem__ = locked(_acquire, _release)(__setitem__)
+
+    def __delitem__(self, key):
+        super(Cache, self).__delitem__(key)
+        self._usage.remove(key)
+    __delitem__ = locked(_acquire, _release)(__delitem__)
+
+    def clear(self):
+        super(Cache, self).clear()
+        self._usage = []
+    clear = locked(_acquire, _release)(clear)
+
+    def pop(self, key, default=_marker):
+        if key in self:
+            self._usage.remove(key)
+        #if default is _marker:
+        #    return super(Cache, self).pop(key)
+        return super(Cache, self).pop(key, default)
+    pop = locked(_acquire, _release)(pop)
+
+    def popitem(self):
+        raise NotImplementedError()
+
+    def setdefault(self, key, default=None):
+        raise NotImplementedError()
+
+    def update(self, other):
+        raise NotImplementedError()
+
+

pylibs/logilab/common/changelog.py

@@ -0,0 +1,236 @@
+# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+#
+# This file is part of logilab-common.
+#
+# logilab-common is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation, either version 2.1 of the License, or (at your option) any
+# later version.
+#
+# logilab-common is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with logilab-common.  If not, see <http://www.gnu.org/licenses/>.
+"""Manipulation of upstream change log files.
+
+The upstream change log files format handled is simpler than the one
+often used such as those generated by the default Emacs changelog mode.
+
+Sample ChangeLog format::
+
+  Change log for project Yoo
+  ==========================
+
+   --
+      * add a new functionality
+
+  2002-02-01 -- 0.1.1
+      * fix bug #435454
+      * fix bug #434356
+
+  2002-01-01 -- 0.1
+      * initial release
+
+
+There is 3 entries in this change log, one for each released version and one
+for the next version (i.e. the current entry).
+Each entry contains a set of messages corresponding to changes done in this
+release.
+All the non empty lines before the first entry are considered as the change
+log title.
+"""
+
+__docformat__ = "restructuredtext en"
+
+import sys
+from stat import S_IWRITE
+
+BULLET = '*'
+SUBBULLET = '-'
+INDENT = ' ' * 4
+
+class NoEntry(Exception):
+    """raised when we are unable to find an entry"""
+
+class EntryNotFound(Exception):
+    """raised when we are unable to find a given entry"""
+
+class Version(tuple):
+    """simple class to handle soft version number has a tuple while
+    correctly printing it as X.Y.Z
+    """
+    def __new__(cls, versionstr):
+        if isinstance(versionstr, basestring):
+            versionstr = versionstr.strip(' :') # XXX (syt) duh?
+            parsed = cls.parse(versionstr)
+        else:
+            parsed = versionstr
+        return tuple.__new__(cls, parsed)
+
+    @classmethod
+    def parse(cls, versionstr):
+        versionstr = versionstr.strip(' :')
+        try:
+            return [int(i) for i in versionstr.split('.')]
+        except ValueError, ex:
+            raise ValueError("invalid literal for version '%s' (%s)"%(versionstr, ex))
+
+    def __str__(self):
+        return '.'.join([str(i) for i in self])
+
+# upstream change log #########################################################
+
+class ChangeLogEntry(object):
+    """a change log entry, i.e. a set of messages associated to a version and
+    its release date
+    """
+    version_class = Version
+
+    def __init__(self, date=None, version=None, **kwargs):
+        self.__dict__.update(kwargs)
+        if version:
+            self.version = self.version_class(version)
+        else:
+            self.version = None
+        self.date = date
+        self.messages = []
+
+    def add_message(self, msg):
+        """add a new message"""
+        self.messages.append(([msg], []))
+
+    def complete_latest_message(self, msg_suite):
+        """complete the latest added message
+        """
+        if not self.messages:
+            raise ValueError('unable to complete last message as there is no previous message)')
+        if self.messages[-1][1]: # sub messages
+            self.messages[-1][1][-1].append(msg_suite)
+        else: # message
+            self.messages[-1][0].append(msg_suite)
+
+    def add_sub_message(self, sub_msg, key=None):
+        if not self.messages:
+            raise ValueError('unable to complete last message as there is no previous message)')
+        if key is None:
+            self.messages[-1][1].append([sub_msg])
+        else:
+            raise NotImplementedError("sub message to specific key are not implemented yet")
+
+    def write(self, stream=sys.stdout):
+        """write the entry to file """
+        stream.write('%s  --  %s\n' % (self.date or '', self.version or ''))
+        for msg, sub_msgs in self.messages:
+            stream.write('%s%s %s\n' % (INDENT, BULLET, msg[0]))
+            stream.write(''.join(msg[1:]))
+            if sub_msgs:
+                stream.write('\n')
+            for sub_msg in sub_msgs:
+                stream.write('%s%s %s\n' % (INDENT * 2, SUBBULLET, sub_msg[0]))
+                stream.write(''.join(sub_msg[1:]))
+            stream.write('\n')
+
+        stream.write('\n\n')
+
+class ChangeLog(object):
+    """object representation of a whole ChangeLog file"""
+
+    entry_class = ChangeLogEntry
+
+    def __init__(self, changelog_file, title=''):
+        self.file = changelog_file
+        self.title = title
+        self.additional_content = ''
+        self.entries = []
+        self.load()
+
+    def __repr__(self):
+        return '<ChangeLog %s at %s (%s entries)>' % (self.file, id(self),
+                                                      len(self.entries))
+
+    def add_entry(self, entry):
+        """add a new entry to the change log"""
+        self.entries.append(entry)
+
+    def get_entry(self, version='', create=None):
+        """ return a given changelog entry
+        if version is omitted, return the current entry
+        """
+        if not self.entries:
+            if version or not create:
+                raise NoEntry()
+            self.entries.append(self.entry_class())
+        if not version:
+            if self.entries[0].version and create is not None:
+                self.entries.insert(0, self.entry_class())
+            return self.entries[0]
+        version = self.version_class(version)
+        for entry in self.entries:
+            if entry.version == version:
+                return entry
+        raise EntryNotFound()
+
+    def add(self, msg, create=None):
+        """add a new message to the latest opened entry"""
+        entry = self.get_entry(create=create)
+        entry.add_message(msg)
+
+    def load(self):
+        """ read a logilab's ChangeLog from file """
+        try:
+            stream = open(self.file)
+        except IOError:
+            return
+        last = None
+        expect_sub = False
+        for line in stream.readlines():
+            sline = line.strip()
+            words = sline.split()
+            # if new entry
+            if len(words) == 1 and words[0] == '--':
+                expect_sub = False
+                last = self.entry_class()
+                self.add_entry(last)
+            # if old entry
+            elif len(words) == 3 and words[1] == '--':
+                expect_sub = False
+                last = self.entry_class(words[0], words[2])
+                self.add_entry(last)
+            # if title
+            elif sline and last is None:
+                self.title = '%s%s' % (self.title, line)
+            # if new entry
+            elif sline and sline[0] == BULLET:
+                expect_sub = False
+                last.add_message(sline[1:].strip())
+            # if new sub_entry
+            elif expect_sub and sline and sline[0] == SUBBULLET:
+                last.add_sub_message(sline[1:].strip())
+            # if new line for current entry
+            elif sline and last.messages:
+                last.complete_latest_message(line)
+            else:
+                expect_sub = True
+                self.additional_content += line
+        stream.close()
+
+    def format_title(self):
+        return '%s\n\n' % self.title.strip()
+
+    def save(self):
+        """write back change log"""
+        # filetutils isn't importable in appengine, so import locally
+        from logilab.common.fileutils import ensure_fs_mode
+        ensure_fs_mode(self.file, S_IWRITE)
+        self.write(open(self.file, 'w'))
+
+    def write(self, stream=sys.stdout):
+        """write changelog to stream"""
+        stream.write(self.format_title())
+        for entry in self.entries:
+            entry.write(stream)
+

pylibs/logilab/common/clcommands.py

@@ -0,0 +1,332 @@
+# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+#
+# This file is part of logilab-common.
+#
+# logilab-common is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation, either version 2.1 of the License, or (at your option) any
+# later version.
+#
+# logilab-common is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with logilab-common.  If not, see <http://www.gnu.org/licenses/>.
+"""Helper functions to support command line tools providing more than
+one command.
+
+e.g called as "tool command [options] args..." where <options> and <args> are
+command'specific
+"""
+
+__docformat__ = "restructuredtext en"
+
+import sys
+import logging
+from os.path import basename
+
+from logilab.common.configuration import Configuration
+from logilab.common.logging_ext import init_log, get_threshold
+from logilab.common.deprecation import deprecated
+
+
+class BadCommandUsage(Exception):
+    """Raised when an unknown command is used or when a command is not
+    correctly used (bad options, too much / missing arguments...).
+
+    Trigger display of command usage.
+    """
+
+class CommandError(Exception):
+    """Raised when a command can't be processed and we want to display it and
+    exit, without traceback nor usage displayed.
+    """
+
+
+# command line access point ####################################################
+
+class CommandLine(dict):
+    """Usage:
+
+    >>> LDI = cli.CommandLine('ldi', doc='Logilab debian installer',
+                              version=version, rcfile=RCFILE)
+    >>> LDI.register(MyCommandClass)
+    >>> LDI.register(MyOtherCommandClass)
+    >>> LDI.run(sys.argv[1:])
+
+    Arguments:
+
+    * `pgm`, the program name, default to `basename(sys.argv[0])`
+
+    * `doc`, a short description of the command line tool
+
+    * `copyright`, additional doc string that will be appended to the generated
+      doc
+
+    * `version`, version number of string of the tool. If specified, global
+      --version option will be available.
+
+    * `rcfile`, path to a configuration file. If specified, global --C/--rc-file
+      option will be available?  self.rcfile = rcfile
+
+    * `logger`, logger to propagate to commands, default to
+      `logging.getLogger(self.pgm))`
+    """
+    def __init__(self, pgm=None, doc=None, copyright=None, version=None,
+                 rcfile=None, logthreshold=logging.ERROR,
+                 check_duplicated_command=True):
+        if pgm is None:
+            pgm = basename(sys.argv[0])
+        self.pgm = pgm
+        self.doc = doc
+        self.copyright = copyright
+        self.version = version
+        self.rcfile = rcfile
+        self.logger = None
+        self.logthreshold = logthreshold
+        self.check_duplicated_command = check_duplicated_command
+
+    def register(self, cls, force=False):
+        """register the given :class:`Command` subclass"""
+        assert not self.check_duplicated_command or force or not cls.name in self, \
+               'a command %s is already defined' % cls.name
+        self[cls.name] = cls
+        return cls
+
+    def run(self, args):
+        """main command line access point:
+        * init logging
+        * handle global options (-h/--help, --version, -C/--rc-file)
+        * check command
+        * run command
+
+        Terminate by :exc:`SystemExit`
+        """
+        init_log(debug=True, # so that we use StreamHandler
+                 logthreshold=self.logthreshold,
+                 logformat='%(levelname)s: %(message)s')
+        try:
+            arg = args.pop(0)
+        except IndexError:
+            self.usage_and_exit(1)
+        if arg in ('-h', '--help'):
+            self.usage_and_exit(0)
+        if self.version is not None and arg in ('--version'):
+            print self.version
+            sys.exit(0)
+        rcfile = self.rcfile
+        if rcfile is not None and arg in ('-C', '--rc-file'):
+            try:
+                rcfile = args.pop(0)
+                arg = args.pop(0)
+            except IndexError:
+                self.usage_and_exit(1)
+        try:
+            command = self.get_command(arg)
+        except KeyError:
+            print 'ERROR: no %s command' % arg
+            print
+            self.usage_and_exit(1)
+        try:
+            sys.exit(command.main_run(args, rcfile))
+        except KeyboardInterrupt, exc:
+            print 'Interrupted',
+            if str(exc):
+                print ': %s' % exc,
+            print
+            sys.exit(4)
+        except BadCommandUsage, err:
+            print 'ERROR:', err
+            print
+            print command.help()
+            sys.exit(1)
+
+    def create_logger(self, handler, logthreshold=None):
+        logger = logging.Logger(self.pgm)
+        logger.handlers = [handler]
+        if logthreshold is None:
+            logthreshold = get_threshold(self.logthreshold)
+        logger.setLevel(logthreshold)
+        return logger
+
+    def get_command(self, cmd, logger=None):
+        if logger is None:
+            logger = self.logger
+        if logger is None:
+            logger = self.logger = logging.getLogger(self.pgm)
+            logger.setLevel(get_threshold(self.logthreshold))
+        return self[cmd](logger)
+
+    def usage(self):
+        """display usage for the main program (i.e. when no command supplied)
+        and exit
+        """
+        print 'usage:', self.pgm,
+        if self.rcfile:
+            print '[--rc-file=<configuration file>]',
+        print '<command> [options] <command argument>...'
+        if self.doc:
+            print '\n%s' % self.doc
+        print  '''
+Type "%(pgm)s <command> --help" for more information about a specific
+command. Available commands are :\n''' % self.__dict__
+        max_len = max([len(cmd) for cmd in self])
+        padding = ' ' * max_len
+        for cmdname, cmd in sorted(self.items()):
+            if not cmd.hidden:
+                print ' ', (cmdname + padding)[:max_len], cmd.short_description()
+        if self.rcfile:
+            print '''
+Use --rc-file=<configuration file> / -C <configuration file> before the command
+to specify a configuration file. Default to %s.
+''' % self.rcfile
+        print  '''%(pgm)s -h/--help
+      display this usage information and exit''' % self.__dict__
+        if self.version:
+            print  '''%(pgm)s -v/--version
+      display version configuration and exit''' % self.__dict__
+        if self.copyright:
+            print '\n', self.copyright
+
+    def usage_and_exit(self, status):
+        self.usage()
+        sys.exit(status)
+
+
+# base command classes #########################################################
+
+class Command(Configuration):
+    """Base class for command line commands.
+
+    Class attributes:
+
+    * `name`, the name of the command
+
+    * `min_args`, minimum number of arguments, None if unspecified
+
+    * `max_args`, maximum number of arguments, None if unspecified
+
+    * `arguments`, string describing arguments, used in command usage
+
+    * `hidden`, boolean flag telling if the command should be hidden, e.g. does
+      not appear in help's commands list
+
+    * `options`, options list, as allowed by :mod:configuration
+    """
+
+    arguments = ''
+    name = ''
+    # hidden from help ?
+    hidden = False
+    # max/min args, None meaning unspecified
+    min_args = None
+    max_args = None
+
+    @classmethod
+    def description(cls):
+        return cls.__doc__.replace('    ', '')
+
+    @classmethod
+    def short_description(cls):
+        return cls.description().split('.')[0]
+
+    def __init__(self, logger):
+        usage = '%%prog %s %s\n\n%s' % (self.name, self.arguments,
+                                        self.description())
+        Configuration.__init__(self, usage=usage)
+        self.logger = logger
+
+    def check_args(self, args):
+        """check command's arguments are provided"""
+        if self.min_args is not None and len(args) < self.min_args:
+            raise BadCommandUsage('missing argument')
+        if self.max_args is not None and len(args) > self.max_args:
+            raise BadCommandUsage('too many arguments')
+
+    def main_run(self, args, rcfile=None):
+        """Run the command and return status 0 if everything went fine.
+
+        If :exc:`CommandError` is raised by the underlying command, simply log
+        the error and return status 2.
+
+        Any other exceptions, including :exc:`BadCommandUsage` will be
+        propagated.
+        """
+        if rcfile:
+            self.load_file_configuration(rcfile)
+        args = self.load_command_line_configuration(args)
+        try:
+            self.check_args(args)
+            self.run(args)
+        except CommandError, err:
+            self.logger.error(err)
+            return 2
+        return 0
+
+    def run(self, args):
+        """run the command with its specific arguments"""
+        raise NotImplementedError()
+
+
+class ListCommandsCommand(Command):
+    """list available commands, useful for bash completion."""
+    name = 'listcommands'
+    arguments = '[command]'
+    hidden = True
+
+    def run(self, args):
+        """run the command with its specific arguments"""
+        if args:
+            command = args.pop()
+            cmd = _COMMANDS[command]
+            for optname, optdict in cmd.options:
+                print '--help'
+                print '--' + optname
+        else:
+            commands = sorted(_COMMANDS.keys())
+            for command in commands:
+                cmd = _COMMANDS[command]
+                if not cmd.hidden:
+                    print command
+
+
+# deprecated stuff #############################################################
+
+_COMMANDS = CommandLine()
+
+DEFAULT_COPYRIGHT = '''\
+Copyright (c) 2004-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+http://www.logilab.fr/ -- mailto:contact@logilab.fr'''
+
+@deprecated('use cls.register(cli)')
+def register_commands(commands):
+    """register existing commands"""
+    for command_klass in commands:
+        _COMMANDS.register(command_klass)
+
+@deprecated('use args.pop(0)')
+def main_run(args, doc=None, copyright=None, version=None):
+    """command line tool: run command specified by argument list (without the
+    program name). Raise SystemExit with status 0 if everything went fine.
+
+    >>> main_run(sys.argv[1:])
+    """
+    _COMMANDS.doc = doc
+    _COMMANDS.copyright = copyright
+    _COMMANDS.version = version
+    _COMMANDS.run(args)
+
+@deprecated('use args.pop(0)')
+def pop_arg(args_list, expected_size_after=None, msg="Missing argument"):
+    """helper function to get and check command line arguments"""
+    try:
+        value = args_list.pop(0)
+    except IndexError:
+        raise BadCommandUsage(msg)
+    if expected_size_after is not None and len(args_list) > expected_size_after:
+        raise BadCommandUsage('too many arguments')
+    return value
+

pylibs/logilab/common/cli.py

@@ -0,0 +1,208 @@
+# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+#
+# This file is part of logilab-common.
+#
+# logilab-common is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation, either version 2.1 of the License, or (at your option) any
+# later version.
+#
+# logilab-common is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with logilab-common.  If not, see <http://www.gnu.org/licenses/>.
+"""Command line interface helper classes.
+
+It provides some default commands, a help system, a default readline
+configuration with completion and persistent history.
+
+Example::
+
+    class BookShell(CLIHelper):
+
+        def __init__(self):
+            # quit and help are builtins
+            # CMD_MAP keys are commands, values are topics
+            self.CMD_MAP['pionce'] = _("Sommeil")
+            self.CMD_MAP['ronfle'] = _("Sommeil")
+            CLIHelper.__init__(self)
+
+        help_do_pionce = ("pionce", "pionce duree", _("met ton corps en veille"))
+        def do_pionce(self):
+            print 'nap is good'
+
+        help_do_ronfle = ("ronfle", "ronfle volume", _("met les autres en veille"))
+        def do_ronfle(self):
+            print 'fuuuuuuuuuuuu rhhhhhrhrhrrh'
+
+    cl = BookShell()
+"""
+
+__docformat__ = "restructuredtext en"
+
+from logilab.common.compat import raw_input, builtins
+if not hasattr(builtins, '_'):
+    builtins._ = str
+
+
+def init_readline(complete_method, histfile=None):
+    """Init the readline library if available."""
+    try:
+        import readline
+        readline.parse_and_bind("tab: complete")
+        readline.set_completer(complete_method)
+        string = readline.get_completer_delims().replace(':', '')
+        readline.set_completer_delims(string)
+        if histfile is not None:
+            try:
+                readline.read_history_file(histfile)
+            except IOError:
+                pass
+            import atexit
+            atexit.register(readline.write_history_file, histfile)
+    except:
+        print 'readline is not available :-('
+
+
+class Completer :
+    """Readline completer."""
+
+    def __init__(self, commands):
+        self.list = commands
+
+    def complete(self, text, state):
+        """Hook called by readline when <tab> is pressed."""
+        n = len(text)
+        matches = []
+        for cmd in self.list :
+            if cmd[:n] == text :
+                matches.append(cmd)
+        try:
+            return matches[state]
+        except IndexError:
+            return None
+
+
+class CLIHelper:
+    """An abstract command line interface client which recognize commands
+    and provide an help system.
+    """
+
+    CMD_MAP = {'help': _("Others"),
+               'quit': _("Others"),
+               }
+    CMD_PREFIX = ''
+
+    def __init__(self, histfile=None) :
+        self._topics = {}
+        self.commands = None
+        self._completer = Completer(self._register_commands())
+        init_readline(self._completer.complete, histfile)
+
+    def run(self):
+        """loop on user input, exit on EOF"""
+        while True:
+            try:
+                line = raw_input('>>> ')
+            except EOFError:
+                print
+                break
+            s_line = line.strip()
+            if not s_line:
+                continue
+            args = s_line.split()
+            if args[0] in self.commands:
+                try:
+                    cmd = 'do_%s' % self.commands[args[0]]
+                    getattr(self, cmd)(*args[1:])
+                except EOFError:
+                    break
+                except:
+                    import traceback
+                    traceback.print_exc()
+            else:
+                try:
+                    self.handle_line(s_line)
+                except:
+                    import traceback
+                    traceback.print_exc()
+
+    def handle_line(self, stripped_line):
+        """Method to overload in the concrete class (should handle
+        lines which are not commands).
+        """
+        raise NotImplementedError()
+
+
+    # private methods #########################################################
+
+    def _register_commands(self):
+        """ register available commands method and return the list of
+        commands name
+        """
+        self.commands = {}
+        self._command_help = {}
+        commands = [attr[3:] for attr in dir(self) if attr[:3] == 'do_']
+        for command in commands:
+            topic = self.CMD_MAP[command]
+            help_method = getattr(self, 'help_do_%s' % command)
+            self._topics.setdefault(topic, []).append(help_method)
+            self.commands[self.CMD_PREFIX + command] = command
+            self._command_help[command] = help_method
+        return self.commands.keys()
+
+    def _print_help(self, cmd, syntax, explanation):
+        print _('Command %s') % cmd
+        print _('Syntax: %s') % syntax
+        print '\t', explanation
+        print
+
+
+    # predefined commands #####################################################
+
+    def do_help(self, command=None) :
+        """base input of the help system"""
+        if command in self._command_help:
+            self._print_help(*self._command_help[command])
+        elif command is None or command not in self._topics:
+            print _("Use help <topic> or help <command>.")
+            print _("Available topics are:")
+            topics = sorted(self._topics.keys())
+            for topic in topics:
+                print '\t', topic
+            print
+            print _("Available commands are:")
+            commands = self.commands.keys()
+            commands.sort()
+            for command in commands:
+                print '\t', command[len(self.CMD_PREFIX):]
+
+        else:
+            print _('Available commands about %s:') % command
+            print
+            for command_help_method in self._topics[command]:
+                try:
+                    if callable(command_help_method):
+                        self._print_help(*command_help_method())
+                    else:
+                        self._print_help(*command_help_method)
+                except:
+                    import traceback
+                    traceback.print_exc()
+                    print 'ERROR in help method %s'% (
+                        command_help_method.func_name)
+
+    help_do_help = ("help", "help [topic|command]",
+                    _("print help message for the given topic/command or \
+available topics when no argument"))
+
+    def do_quit(self):
+        """quit the CLI"""
+        raise EOFError()
+
+    def help_do_quit(self):
+        return ("quit", "quit", _("quit the application"))

pylibs/logilab/common/compat.py

@@ -0,0 +1,243 @@
+# pylint: disable=E0601,W0622,W0611
+# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+#
+# This file is part of logilab-common.
+#
+# logilab-common is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation, either version 2.1 of the License, or (at your option) any
+# later version.
+#
+# logilab-common is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with logilab-common.  If not, see <http://www.gnu.org/licenses/>.
+"""Wrappers around some builtins introduced in python 2.3, 2.4 and
+2.5, making them available in for earlier versions of python.
+
+See another compatibility snippets from other projects:
+
+    :mod:`lib2to3.fixes`
+    :mod:`coverage.backward`
+    :mod:`unittest2.compatibility`
+"""
+
+from __future__ import generators
+
+__docformat__ = "restructuredtext en"
+
+import os
+import sys
+import types
+from warnings import warn
+
+import __builtin__ as builtins # 2to3 will tranform '__builtin__' to 'builtins'
+
+if sys.version_info < (3, 0):
+    str_to_bytes = str
+    def str_encode(string, encoding):
+        if isinstance(string, unicode):
+            return string.encode(encoding)
+        return str(string)
+else:
+    def str_to_bytes(string):
+        return str.encode(string)
+    # we have to ignore the encoding in py3k to be able to write a string into a
+    # TextIOWrapper or like object (which expect an unicode string)
+    def str_encode(string, encoding):
+        return str(string)
+
+# XXX callable built-in seems back in all python versions
+try:
+    callable = builtins.callable
+except AttributeError:
+    from collections import Callable
+    def callable(something):
+        return isinstance(something, Callable)
+    del Callable
+
+# See also http://bugs.python.org/issue11776
+if sys.version_info[0] == 3:
+    def method_type(callable, instance, klass):
+        # api change. klass is no more considered
+        return types.MethodType(callable, instance)
+else:
+    # alias types otherwise
+    method_type = types.MethodType
+
+if sys.version_info < (3, 0):
+    raw_input = raw_input
+else:
+    raw_input = input
+
+# Pythons 2 and 3 differ on where to get StringIO
+if sys.version_info < (3, 0):
+    from cStringIO import StringIO
+    FileIO = file
+    BytesIO = StringIO
+    reload = reload
+else:
+    from io import FileIO, BytesIO, StringIO
+    from imp import reload
+
+# Where do pickles come from?
+try:
+    import cPickle as pickle
+except ImportError:
+    import pickle
+
+from logilab.common.deprecation import deprecated
+
+from itertools import izip, chain, imap
+if sys.version_info < (3, 0):# 2to3 will remove the imports
+    izip = deprecated('izip exists in itertools since py2.3')(izip)
+    imap = deprecated('imap exists in itertools since py2.3')(imap)
+chain = deprecated('chain exists in itertools since py2.3')(chain)
+
+sum = deprecated('sum exists in builtins since py2.3')(sum)
+enumerate = deprecated('enumerate exists in builtins since py2.3')(enumerate)
+frozenset = deprecated('frozenset exists in builtins since py2.4')(frozenset)
+reversed = deprecated('reversed exists in builtins since py2.4')(reversed)
+sorted = deprecated('sorted exists in builtins since py2.4')(sorted)
+max = deprecated('max exists in builtins since py2.4')(max)
+
+
+# Python2.5 builtins
+try:
+    any = any
+    all = all
+except NameError:
+    def any(iterable):
+        """any(iterable) -> bool
+
+        Return True if bool(x) is True for any x in the iterable.
+        """
+        for elt in iterable:
+            if elt:
+                return True
+        return False
+
+    def all(iterable):
+        """all(iterable) -> bool
+
+        Return True if bool(x) is True for all values x in the iterable.
+        """
+        for elt in iterable:
+            if not elt:
+                return False
+        return True
+
+
+# Python2.5 subprocess added functions and exceptions
+try:
+    from subprocess import Popen
+except ImportError:
+    # gae or python < 2.3
+
+    class CalledProcessError(Exception):
+        """This exception is raised when a process run by check_call() returns
+        a non-zero exit status.  The exit status will be stored in the
+        returncode attribute."""
+        def __init__(self, returncode, cmd):
+            self.returncode = returncode
+            self.cmd = cmd
+        def __str__(self):
+            return "Command '%s' returned non-zero exit status %d" % (self.cmd,
+    self.returncode)
+
+    def call(*popenargs, **kwargs):
+        """Run command with arguments.  Wait for command to complete, then
+        return the returncode attribute.
+
+        The arguments are the same as for the Popen constructor.  Example:
+
+        retcode = call(["ls", "-l"])
+        """
+        # workaround: subprocess.Popen(cmd, stdout=sys.stdout) fails
+        # see http://bugs.python.org/issue1531862
+        if "stdout" in kwargs:
+            fileno = kwargs.get("stdout").fileno()
+            del kwargs['stdout']
+            return Popen(stdout=os.dup(fileno), *popenargs, **kwargs).wait()
+        return Popen(*popenargs, **kwargs).wait()
+
+    def check_call(*popenargs, **kwargs):
+        """Run command with arguments.  Wait for command to complete.  If
+        the exit code was zero then return, otherwise raise
+        CalledProcessError.  The CalledProcessError object will have the
+        return code in the returncode attribute.
+
+        The arguments are the same as for the Popen constructor.  Example:
+
+        check_call(["ls", "-l"])
+        """
+        retcode = call(*popenargs, **kwargs)
+        cmd = kwargs.get("args")
+        if cmd is None:
+            cmd = popenargs[0]
+        if retcode:
+            raise CalledProcessError(retcode, cmd)
+        return retcode
+
+try:
+    from os.path import relpath
+except ImportError: # python < 2.6
+    from os.path import curdir, abspath, sep, commonprefix, pardir, join
+    def relpath(path, start=curdir):
+        """Return a relative version of a path"""
+
+        if not path:
+            raise ValueError("no path specified")
+
+        start_list = abspath(start).split(sep)
+        path_list = abspath(path).split(sep)
+
+        # Work out how much of the filepath is shared by start and path.
+        i = len(commonprefix([start_list, path_list]))
+
+        rel_list = [pardir] * (len(start_list)-i) + path_list[i:]
+        if not rel_list:
+            return curdir
+        return join(*rel_list)
+
+
+# XXX don't know why tests don't pass if I don't do that :
+_real_set, set = set, deprecated('set exists in builtins since py2.4')(set)
+if (2, 5) <= sys.version_info[:2]:
+    InheritableSet = _real_set
+else:
+    class InheritableSet(_real_set):
+        """hacked resolving inheritancy issue from old style class in 2.4"""
+        def __new__(cls, *args, **kwargs):
+            if args:
+                new_args = (args[0], )
+            else:
+                new_args = ()
+            obj = _real_set.__new__(cls, *new_args)
+            obj.__init__(*args, **kwargs)
+            return obj
+
+# XXX shouldn't we remove this and just let 2to3 do his job ?
+# range or xrange?
+try:
+    range = xrange
+except NameError:
+    range = range
+
+# ConfigParser was renamed to the more-standard configparser
+try:
+    import configparser
+except ImportError:
+    import ConfigParser as configparser
+
+try:
+    import json
+except ImportError:
+    try:
+        import simplejson as json
+    except ImportError:
+        json = None

pylibs/logilab/common/configuration.py

@@ -0,0 +1,1069 @@
+# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+#
+# This file is part of logilab-common.
+#
+# logilab-common is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation, either version 2.1 of the License, or (at your option) any
+# later version.
+#
+# logilab-common is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with logilab-common.  If not, see <http://www.gnu.org/licenses/>.
+"""Classes to handle advanced configuration in simple to complex applications.
+
+Allows to load the configuration from a file or from command line
+options, to generate a sample configuration file or to display
+program's usage. Fills the gap between optik/optparse and ConfigParser
+by adding data types (which are also available as a standalone optik
+extension in the `optik_ext` module).
+
+
+Quick start: simplest usage
+---------------------------
+
+.. python ::
+
+  >>> import sys
+  >>> from logilab.common.configuration import Configuration
+  >>> options = [('dothis', {'type':'yn', 'default': True, 'metavar': '<y or n>'}),
+  ...            ('value', {'type': 'string', 'metavar': '<string>'}),
+  ...            ('multiple', {'type': 'csv', 'default': ('yop',),
+  ...                          'metavar': '<comma separated values>',
+  ...                          'help': 'you can also document the option'}),
+  ...            ('number', {'type': 'int', 'default':2, 'metavar':'<int>'}),
+  ...           ]
+  >>> config = Configuration(options=options, name='My config')
+  >>> print config['dothis']
+  True
+  >>> print config['value']
+  None
+  >>> print config['multiple']
+  ('yop',)
+  >>> print config['number']
+  2
+  >>> print config.help()
+  Usage:  [options]
+
+  Options:
+    -h, --help            show this help message and exit
+    --dothis=<y or n>
+    --value=<string>
+    --multiple=<comma separated values>
+                          you can also document the option [current: none]
+    --number=<int>
+
+  >>> f = open('myconfig.ini', 'w')
+  >>> f.write('''[MY CONFIG]
+  ... number = 3
+  ... dothis = no
+  ... multiple = 1,2,3
+  ... ''')
+  >>> f.close()
+  >>> config.load_file_configuration('myconfig.ini')
+  >>> print config['dothis']
+  False
+  >>> print config['value']
+  None
+  >>> print config['multiple']
+  ['1', '2', '3']
+  >>> print config['number']
+  3
+  >>> sys.argv = ['mon prog', '--value', 'bacon', '--multiple', '4,5,6',
+  ...             'nonoptionargument']
+  >>> print config.load_command_line_configuration()
+  ['nonoptionargument']
+  >>> print config['value']
+  bacon
+  >>> config.generate_config()
+  # class for simple configurations which don't need the
+  # manager / providers model and prefer delegation to inheritance
+  #
+  # configuration values are accessible through a dict like interface
+  #
+  [MY CONFIG]
+
+  dothis=no
+
+  value=bacon
+
+  # you can also document the option
+  multiple=4,5,6
+
+  number=3
+  >>>
+"""
+__docformat__ = "restructuredtext en"
+
+__all__ = ('OptionsManagerMixIn', 'OptionsProviderMixIn',
+           'ConfigurationMixIn', 'Configuration',
+           'OptionsManager2ConfigurationAdapter')
+
+import os
+import sys
+import re
+from os.path import exists, expanduser
+from copy import copy
+from ConfigParser import ConfigParser, NoOptionError, NoSectionError, \
+     DuplicateSectionError
+from warnings import warn
+
+from logilab.common.compat import callable, raw_input, str_encode as _encode
+
+from logilab.common.textutils import normalize_text, unquote
+from logilab.common import optik_ext as optparse
+
+OptionError = optparse.OptionError
+
+REQUIRED = []
+
+class UnsupportedAction(Exception):
+    """raised by set_option when it doesn't know what to do for an action"""
+
+
+def _get_encoding(encoding, stream):
+    encoding = encoding or getattr(stream, 'encoding', None)
+    if not encoding:
+        import locale
+        encoding = locale.getpreferredencoding()
+    return encoding
+
+
+# validation functions ########################################################
+
+def choice_validator(optdict, name, value):
+    """validate and return a converted value for option of type 'choice'
+    """
+    if not value in optdict['choices']:
+        msg = "option %s: invalid value: %r, should be in %s"
+        raise optparse.OptionValueError(msg % (name, value, optdict['choices']))
+    return value
+
+def multiple_choice_validator(optdict, name, value):
+    """validate and return a converted value for option of type 'choice'
+    """
+    choices = optdict['choices']
+    values = optparse.check_csv(None, name, value)
+    for value in values:
+        if not value in choices:
+            msg = "option %s: invalid value: %r, should be in %s"
+            raise optparse.OptionValueError(msg % (name, value, choices))
+    return values
+
+def csv_validator(optdict, name, value):
+    """validate and return a converted value for option of type 'csv'
+    """
+    return optparse.check_csv(None, name, value)
+
+def yn_validator(optdict, name, value):
+    """validate and return a converted value for option of type 'yn'
+    """
+    return optparse.check_yn(None, name, value)
+
+def named_validator(optdict, name, value):
+    """validate and return a converted value for option of type 'named'
+    """
+    return optparse.check_named(None, name, value)
+
+def file_validator(optdict, name, value):
+    """validate and return a filepath for option of type 'file'"""
+    return optparse.check_file(None, name, value)
+
+def color_validator(optdict, name, value):
+    """validate and return a valid color for option of type 'color'"""
+    return optparse.check_color(None, name, value)
+
+def password_validator(optdict, name, value):
+    """validate and return a string for option of type 'password'"""
+    return optparse.check_password(None, name, value)
+
+def date_validator(optdict, name, value):
+    """validate and return a mx DateTime object for option of type 'date'"""
+    return optparse.check_date(None, name, value)
+
+def time_validator(optdict, name, value):
+    """validate and return a time object for option of type 'time'"""
+    return optparse.check_time(None, name, value)
+
+def bytes_validator(optdict, name, value):
+    """validate and return an integer for option of type 'bytes'"""
+    return optparse.check_bytes(None, name, value)
+
+
+VALIDATORS = {'string': unquote,
+              'int': int,
+              'float': float,
+              'file': file_validator,
+              'font': unquote,
+              'color': color_validator,
+              'regexp': re.compile,
+              'csv': csv_validator,
+              'yn': yn_validator,
+              'bool': yn_validator,
+              'named': named_validator,
+              'password': password_validator,
+              'date': date_validator,
+              'time': time_validator,
+              'bytes': bytes_validator,
+              'choice': choice_validator,
+              'multiple_choice': multiple_choice_validator,
+              }
+
+def _call_validator(opttype, optdict, option, value):
+    if opttype not in VALIDATORS:
+        raise Exception('Unsupported type "%s"' % opttype)
+    try:
+        return VALIDATORS[opttype](optdict, option, value)
+    except TypeError:
+        try:
+            return VALIDATORS[opttype](value)
+        except optparse.OptionValueError:
+            raise
+        except:
+            raise optparse.OptionValueError('%s value (%r) should be of type %s' %
+                                   (option, value, opttype))
+
+# user input functions ########################################################
+
+def input_password(optdict, question='password:'):
+    from getpass import getpass
+    while True:
+        value = getpass(question)
+        value2 = getpass('confirm: ')
+        if value == value2:
+            return value
+        print 'password mismatch, try again'
+
+def input_string(optdict, question):
+    value = raw_input(question).strip()
+    return value or None
+
+def _make_input_function(opttype):
+    def input_validator(optdict, question):
+        while True:
+            value = raw_input(question)
+            if not value.strip():
+                return None
+            try:
+                return _call_validator(opttype, optdict, None, value)
+            except optparse.OptionValueError, ex:
+                msg = str(ex).split(':', 1)[-1].strip()
+                print 'bad value: %s' % msg
+    return input_validator
+
+INPUT_FUNCTIONS = {
+    'string': input_string,
+    'password': input_password,
+    }
+
+for opttype in VALIDATORS.keys():
+    INPUT_FUNCTIONS.setdefault(opttype, _make_input_function(opttype))
+
+def expand_default(self, option):
+    """monkey patch OptionParser.expand_default since we have a particular
+    way to handle defaults to avoid overriding values in the configuration
+    file
+    """
+    if self.parser is None or not self.default_tag:
+        return option.help
+    optname = option._long_opts[0][2:]
+    try:
+        provider = self.parser.options_manager._all_options[optname]
+    except KeyError:
+        value = None
+    else:
+        optdict = provider.get_option_def(optname)
+        optname = provider.option_name(optname, optdict)
+        value = getattr(provider.config, optname, optdict)
+        value = format_option_value(optdict, value)
+    if value is optparse.NO_DEFAULT or not value:
+        value = self.NO_DEFAULT_VALUE
+    return option.help.replace(self.default_tag, str(value))
+
+
+def convert(value, optdict, name=''):
+    """return a validated value for an option according to its type
+
+    optional argument name is only used for error message formatting
+    """
+    try:
+        _type = optdict['type']
+    except KeyError:
+        # FIXME
+        return value
+    return _call_validator(_type, optdict, name, value)
+
+def comment(string):
+    """return string as a comment"""
+    lines = [line.strip() for line in string.splitlines()]
+    return '# ' + ('%s# ' % os.linesep).join(lines)
+
+def format_time(value):
+    if not value:
+        return '0'
+    if value != int(value):
+        return '%.2fs' % value
+    value = int(value)
+    nbmin, nbsec = divmod(value, 60)
+    if nbsec:
+        return '%ss' % value
+    nbhour, nbmin_ = divmod(nbmin, 60)
+    if nbmin_:
+        return '%smin' % nbmin
+    nbday, nbhour_ = divmod(nbhour, 24)
+    if nbhour_:
+        return '%sh' % nbhour
+    return '%sd' % nbday
+
+def format_bytes(value):
+    if not value:
+        return '0'
+    if value != int(value):
+        return '%.2fB' % value
+    value = int(value)
+    prevunit = 'B'
+    for unit in ('KB', 'MB', 'GB', 'TB'):
+        next, remain = divmod(value, 1024)
+        if remain:
+            return '%s%s' % (value, prevunit)
+        prevunit = unit
+        value = next
+    return '%s%s' % (value, unit)
+
+def format_option_value(optdict, value):
+    """return the user input's value from a 'compiled' value"""
+    if isinstance(value, (list, tuple)):
+        value = ','.join(value)
+    elif isinstance(value, dict):
+        value = ','.join(['%s:%s' % (k, v) for k, v in value.items()])
+    elif hasattr(value, 'match'): # optdict.get('type') == 'regexp'
+        # compiled regexp
+        value = value.pattern
+    elif optdict.get('type') == 'yn':
+        value = value and 'yes' or 'no'
+    elif isinstance(value, (str, unicode)) and value.isspace():
+        value = "'%s'" % value
+    elif optdict.get('type') == 'time' and isinstance(value, (float, int, long)):
+        value = format_time(value)
+    elif optdict.get('type') == 'bytes' and hasattr(value, '__int__'):
+        value = format_bytes(value)
+    return value
+
+def ini_format_section(stream, section, options, encoding=None, doc=None):
+    """format an options section using the INI format"""
+    encoding = _get_encoding(encoding, stream)
+    if doc:
+        print >> stream, _encode(comment(doc), encoding)
+    print >> stream, '[%s]' % section
+    ini_format(stream, options, encoding)
+
+def ini_format(stream, options, encoding):
+    """format options using the INI format"""
+    for optname, optdict, value in options:
+        value = format_option_value(optdict, value)
+        help = optdict.get('help')
+        if help:
+            help = normalize_text(help, line_len=79, indent='# ')
+            print >> stream
+            print >> stream, _encode(help, encoding)
+        else:
+            print >> stream
+        if value is None:
+            print >> stream, '#%s=' % optname
+        else:
+            value = _encode(value, encoding).strip()
+            print >> stream, '%s=%s' % (optname, value)
+
+format_section = ini_format_section
+
+def rest_format_section(stream, section, options, encoding=None, doc=None):
+    """format an options section using the INI format"""
+    encoding = _get_encoding(encoding, stream)
+    if section:
+        print >> stream, '%s\n%s' % (section, "'"*len(section))
+    if doc:
+        print >> stream, _encode(normalize_text(doc, line_len=79, indent=''),
+                                 encoding)
+        print >> stream
+    for optname, optdict, value in options:
+        help = optdict.get('help')
+        print >> stream, ':%s:' % optname
+        if help:
+            help = normalize_text(help, line_len=79, indent='  ')
+            print >> stream, _encode(help, encoding)
+        if value:
+            value = _encode(format_option_value(optdict, value), encoding)
+            print >> stream, ''
+            print >> stream, '  Default: ``%s``' % value.replace("`` ", "```` ``")
+
+
+class OptionsManagerMixIn(object):
+    """MixIn to handle a configuration from both a configuration file and
+    command line options
+    """
+
+    def __init__(self, usage, config_file=None, version=None, quiet=0):
+        self.config_file = config_file
+        self.reset_parsers(usage, version=version)
+        # list of registered options providers
+        self.options_providers = []
+        # dictionary associating option name to checker
+        self._all_options = {}
+        self._short_options = {}
+        self._nocallback_options = {}
+        self._mygroups = dict()
+        # verbosity
+        self.quiet = quiet
+        self._maxlevel = 0
+
+    def reset_parsers(self, usage='', version=None):
+        # configuration file parser
+        self.cfgfile_parser = ConfigParser()
+        # command line parser
+        self.cmdline_parser = optparse.OptionParser(usage=usage, version=version)
+        self.cmdline_parser.options_manager = self
+        self._optik_option_attrs = set(self.cmdline_parser.option_class.ATTRS)
+
+    def register_options_provider(self, provider, own_group=True):
+        """register an options provider"""
+        assert provider.priority <= 0, "provider's priority can't be >= 0"
+        for i in range(len(self.options_providers)):
+            if provider.priority > self.options_providers[i].priority:
+                self.options_providers.insert(i, provider)
+                break
+        else:
+            self.options_providers.append(provider)
+        non_group_spec_options = [option for option in provider.options
+                                  if 'group' not in option[1]]
+        groups = getattr(provider, 'option_groups', ())
+        if own_group and non_group_spec_options:
+            self.add_option_group(provider.name.upper(), provider.__doc__,
+                                  non_group_spec_options, provider)
+        else:
+            for opt, optdict in non_group_spec_options:
+                self.add_optik_option(provider, self.cmdline_parser, opt, optdict)
+        for gname, gdoc in groups:
+            gname = gname.upper()
+            goptions = [option for option in provider.options
+                        if option[1].get('group', '').upper() == gname]
+            self.add_option_group(gname, gdoc, goptions, provider)
+
+    def add_option_group(self, group_name, doc, options, provider):
+        """add an option group including the listed options
+        """
+        assert options
+        # add option group to the command line parser
+        if group_name in self._mygroups:
+            group = self._mygroups[group_name]
+        else:
+            group = optparse.OptionGroup(self.cmdline_parser,
+                                         title=group_name.capitalize())
+            self.cmdline_parser.add_option_group(group)
+            group.level = provider.level
+            self._mygroups[group_name] = group
+            # add section to the config file
+            if group_name != "DEFAULT":
+                self.cfgfile_parser.add_section(group_name)
+        # add provider's specific options
+        for opt, optdict in options:
+            self.add_optik_option(provider, group, opt, optdict)
+
+    def add_optik_option(self, provider, optikcontainer, opt, optdict):
+        if 'inputlevel' in optdict:
+            warn('[0.50] "inputlevel" in option dictionary for %s is deprecated,'
+                 ' use "level"' % opt, DeprecationWarning)
+            optdict['level'] = optdict.pop('inputlevel')
+        args, optdict = self.optik_option(provider, opt, optdict)
+        option = optikcontainer.add_option(*args, **optdict)
+        self._all_options[opt] = provider
+        self._maxlevel = max(self._maxlevel, option.level or 0)
+
+    def optik_option(self, provider, opt, optdict):
+        """get our personal option definition and return a suitable form for
+        use with optik/optparse
+        """
+        optdict = copy(optdict)
+        others = {}
+        if 'action' in optdict:
+            self._nocallback_options[provider] = opt
+        else:
+            optdict['action'] = 'callback'
+            optdict['callback'] = self.cb_set_provider_option
+        # default is handled here and *must not* be given to optik if you
+        # want the whole machinery to work
+        if 'default' in optdict:
+            if (optparse.OPTPARSE_FORMAT_DEFAULT and 'help' in optdict and
+                optdict.get('default') is not None and
+                not optdict['action'] in ('store_true', 'store_false')):
+                optdict['help'] += ' [current: %default]'
+            del optdict['default']
+        args = ['--' + str(opt)]
+        if 'short' in optdict:
+            self._short_options[optdict['short']] = opt
+            args.append('-' + optdict['short'])
+            del optdict['short']
+        # cleanup option definition dict before giving it to optik
+        for key in optdict.keys():
+            if not key in self._optik_option_attrs:
+                optdict.pop(key)
+        return args, optdict
+
+    def cb_set_provider_option(self, option, opt, value, parser):
+        """optik callback for option setting"""
+        if opt.startswith('--'):
+            # remove -- on long option
+            opt = opt[2:]
+        else:
+            # short option, get its long equivalent
+            opt = self._short_options[opt[1:]]
+        # trick since we can't set action='store_true' on options
+        if value is None:
+            value = 1
+        self.global_set_option(opt, value)
+
+    def global_set_option(self, opt, value):
+        """set option on the correct option provider"""
+        self._all_options[opt].set_option(opt, value)
+
+    def generate_config(self, stream=None, skipsections=(), encoding=None):
+        """write a configuration file according to the current configuration
+        into the given stream or stdout
+        """
+        options_by_section = {}
+        sections = []
+        for provider in self.options_providers:
+            for section, options in provider.options_by_section():
+                if section is None:
+                    section = provider.name
+                if section in skipsections:
+                    continue
+                options = [(n, d, v) for (n, d, v) in options
+                           if d.get('type') is not None]
+                if not options:
+                    continue
+                if not section in sections:
+                    sections.append(section)
+                alloptions = options_by_section.setdefault(section, [])
+                alloptions += options
+        stream = stream or sys.stdout
+        encoding = _get_encoding(encoding, stream)
+        printed = False
+        for section in sections:
+            if printed:
+                print >> stream, '\n'
+            format_section(stream, section.upper(), options_by_section[section],
+                           encoding)
+            printed = True
+
+    def generate_manpage(self, pkginfo, section=1, stream=None):
+        """write a man page for the current configuration into the given
+        stream or stdout
+        """
+        self._monkeypatch_expand_default()
+        try:
+            optparse.generate_manpage(self.cmdline_parser, pkginfo,
+                                      section, stream=stream or sys.stdout,
+                                      level=self._maxlevel)
+        finally:
+            self._unmonkeypatch_expand_default()
+
+    # initialization methods ##################################################
+
+    def load_provider_defaults(self):
+        """initialize configuration using default values"""
+        for provider in self.options_providers:
+            provider.load_defaults()
+
+    def load_file_configuration(self, config_file=None):
+        """load the configuration from file"""
+        self.read_config_file(config_file)
+        self.load_config_file()
+
+    def read_config_file(self, config_file=None):
+        """read the configuration file but do not load it (i.e. dispatching
+        values to each options provider)
+        """
+        helplevel = 1
+        while helplevel <= self._maxlevel:
+            opt = '-'.join(['long'] * helplevel) + '-help'
+            if opt in self._all_options:
+                break # already processed
+            def helpfunc(option, opt, val, p, level=helplevel):
+                print self.help(level)
+                sys.exit(0)
+            helpmsg = '%s verbose help.' % ' '.join(['more'] * helplevel)
+            optdict = {'action' : 'callback', 'callback' : helpfunc,
+                       'help' : helpmsg}
+            provider = self.options_providers[0]
+            self.add_optik_option(provider, self.cmdline_parser, opt, optdict)
+            provider.options += ( (opt, optdict), )
+            helplevel += 1
+        if config_file is None:
+            config_file = self.config_file
+        if config_file is not None:
+            config_file = expanduser(config_file)
+        if config_file and exists(config_file):
+            parser = self.cfgfile_parser
+            parser.read([config_file])
+            # normalize sections'title
+            for sect, values in parser._sections.items():
+                if not sect.isupper() and values:
+                    parser._sections[sect.upper()] = values
+        elif not self.quiet:
+            msg = 'No config file found, using default configuration'
+            print >> sys.stderr, msg
+            return
+
+    def input_config(self, onlysection=None, inputlevel=0, stream=None):
+        """interactively get configuration values by asking to the user and generate
+        a configuration file
+        """
+        if onlysection is not None:
+            onlysection = onlysection.upper()
+        for provider in self.options_providers:
+            for section, option, optdict in provider.all_options():
+                if onlysection is not None and section != onlysection:
+                    continue
+                if not 'type' in optdict:
+                    # ignore action without type (callback, store_true...)
+                    continue
+                provider.input_option(option, optdict, inputlevel)
+        # now we can generate the configuration file
+        if stream is not None:
+            self.generate_config(stream)
+
+    def load_config_file(self):
+        """dispatch values previously read from a configuration file to each
+        options provider)
+        """
+        parser = self.cfgfile_parser
+        for provider in self.options_providers:
+            for section, option, optdict in provider.all_options():
+                try:
+                    value = parser.get(section, option)
+                    provider.set_option(option, value, optdict=optdict)
+                except (NoSectionError, NoOptionError), ex:
+                    continue
+
+    def load_configuration(self, **kwargs):
+        """override configuration according to given parameters
+        """
+        for opt, opt_value in kwargs.items():
+            opt = opt.replace('_', '-')
+            provider = self._all_options[opt]
+            provider.set_option(opt, opt_value)
+
+    def load_command_line_configuration(self, args=None):
+        """override configuration according to command line parameters
+
+        return additional arguments
+        """
+        self._monkeypatch_expand_default()
+        try:
+            if args is None:
+                args = sys.argv[1:]
+            else:
+                args = list(args)
+            (options, args) = self.cmdline_parser.parse_args(args=args)
+            for provider in self._nocallback_options.keys():
+                config = provider.config
+                for attr in config.__dict__.keys():
+                    value = getattr(options, attr, None)
+                    if value is None:
+                        continue
+                    setattr(config, attr, value)
+            return args
+        finally:
+            self._unmonkeypatch_expand_default()
+
+
+    # help methods ############################################################
+
+    def add_help_section(self, title, description, level=0):
+        """add a dummy option section for help purpose """
+        group = optparse.OptionGroup(self.cmdline_parser,
+                                     title=title.capitalize(),
+                                     description=description)
+        group.level = level
+        self._maxlevel = max(self._maxlevel, level)
+        self.cmdline_parser.add_option_group(group)
+
+    def _monkeypatch_expand_default(self):
+        # monkey patch optparse to deal with our default values
+        try:
+            self.__expand_default_backup = optparse.HelpFormatter.expand_default
+            optparse.HelpFormatter.expand_default = expand_default
+        except AttributeError:
+            # python < 2.4: nothing to be done
+            pass
+    def _unmonkeypatch_expand_default(self):
+        # remove monkey patch
+        if hasattr(optparse.HelpFormatter, 'expand_default'):
+            # unpatch optparse to avoid side effects
+            optparse.HelpFormatter.expand_default = self.__expand_default_backup
+
+    def help(self, level=0):
+        """return the usage string for available options """
+        self.cmdline_parser.formatter.output_level = level
+        self._monkeypatch_expand_default()
+        try:
+            return self.cmdline_parser.format_help()
+        finally:
+            self._unmonkeypatch_expand_default()
+
+
+class Method(object):
+    """used to ease late binding of default method (so you can define options
+    on the class using default methods on the configuration instance)
+    """
+    def __init__(self, methname):
+        self.method = methname
+        self._inst = None
+
+    def bind(self, instance):
+        """bind the method to its instance"""
+        if self._inst is None:
+            self._inst = instance
+
+    def __call__(self, *args, **kwargs):
+        assert self._inst, 'unbound method'
+        return getattr(self._inst, self.method)(*args, **kwargs)
+
+
+class OptionsProviderMixIn(object):
+    """Mixin to provide options to an OptionsManager"""
+
+    # those attributes should be overridden
+    priority = -1
+    name = 'default'
+    options = ()
+    level = 0
+
+    def __init__(self):
+        self.config = optparse.Values()
+        for option in self.options:
+            try:
+                option, optdict = option
+            except ValueError:
+                raise Exception('Bad option: %r' % option)
+            if isinstance(optdict.get('default'), Method):
+                optdict['default'].bind(self)
+            elif isinstance(optdict.get('callback'), Method):
+                optdict['callback'].bind(self)
+        self.load_defaults()
+
+    def load_defaults(self):
+        """initialize the provider using default values"""
+        for opt, optdict in self.options:
+            action = optdict.get('action')
+            if action != 'callback':
+                # callback action have no default
+                default = self.option_default(opt, optdict)
+                if default is REQUIRED:
+                    continue
+                self.set_option(opt, default, action, optdict)
+
+    def option_default(self, opt, optdict=None):
+        """return the default value for an option"""
+        if optdict is None:
+            optdict = self.get_option_def(opt)
+        default = optdict.get('default')
+        if callable(default):
+            default = default()
+        return default
+
+    def option_name(self, opt, optdict=None):
+        """get the config attribute corresponding to opt
+        """
+        if optdict is None:
+            optdict = self.get_option_def(opt)
+        return optdict.get('dest', opt.replace('-', '_'))
+
+    def option_value(self, opt):
+        """get the current value for the given option"""
+        return getattr(self.config, self.option_name(opt), None)
+
+    def set_option(self, opt, value, action=None, optdict=None):
+        """method called to set an option (registered in the options list)
+        """
+        # print "************ setting option", opt," to value", value
+        if optdict is None:
+            optdict = self.get_option_def(opt)
+        if value is not None:
+            value = convert(value, optdict, opt)
+        if action is None:
+            action = optdict.get('action', 'store')
+        if optdict.get('type') == 'named': # XXX need specific handling
+            optname = self.option_name(opt, optdict)
+            currentvalue = getattr(self.config, optname, None)
+            if currentvalue:
+                currentvalue.update(value)
+                value = currentvalue
+        if action == 'store':
+            setattr(self.config, self.option_name(opt, optdict), value)
+        elif action in ('store_true', 'count'):
+            setattr(self.config, self.option_name(opt, optdict), 0)
+        elif action == 'store_false':
+            setattr(self.config, self.option_name(opt, optdict), 1)
+        elif action == 'append':
+            opt = self.option_name(opt, optdict)
+            _list = getattr(self.config, opt, None)
+            if _list is None:
+                if isinstance(value, (list, tuple)):
+                    _list = value
+                elif value is not None:
+                    _list = []
+                    _list.append(value)
+                setattr(self.config, opt, _list)
+            elif isinstance(_list, tuple):
+                setattr(self.config, opt, _list + (value,))
+            else:
+                _list.append(value)
+        elif action == 'callback':
+            optdict['callback'](None, opt, value, None)
+        else:
+            raise UnsupportedAction(action)
+
+    def input_option(self, option, optdict, inputlevel=99):
+        default = self.option_default(option, optdict)
+        if default is REQUIRED:
+            defaultstr = '(required): '
+        elif optdict.get('level', 0) > inputlevel:
+            return
+        elif optdict['type'] == 'password' or default is None:
+            defaultstr = ': '
+        else:
+            defaultstr = '(default: %s): ' % format_option_value(optdict, default)
+        print ':%s:' % option
+        print optdict.get('help') or option
+        inputfunc = INPUT_FUNCTIONS[optdict['type']]
+        value = inputfunc(optdict, defaultstr)
+        while default is REQUIRED and not value:
+            print 'please specify a value'
+            value = inputfunc(optdict, '%s: ' % option)
+        if value is None and default is not None:
+            value = default
+        self.set_option(option, value, optdict=optdict)
+
+    def get_option_def(self, opt):
+        """return the dictionary defining an option given it's name"""
+        assert self.options
+        for option in self.options:
+            if option[0] == opt:
+                return option[1]
+        raise OptionError('no such option %s in section %r'
+                          % (opt, self.name), opt)
+
+
+    def all_options(self):
+        """return an iterator on available options for this provider
+        option are actually described by a 3-uple:
+        (section, option name, option dictionary)
+        """
+        for section, options in self.options_by_section():
+            if section is None:
+                if self.name is None:
+                    continue
+                section = self.name.upper()
+            for option, optiondict, value in options:
+                yield section, option, optiondict
+
+    def options_by_section(self):
+        """return an iterator on options grouped by section
+
+        (section, [list of (optname, optdict, optvalue)])
+        """
+        sections = {}
+        for optname, optdict in self.options:
+            sections.setdefault(optdict.get('group'), []).append(
+                (optname, optdict, self.option_value(optname)))
+        if None in sections:
+            yield None, sections.pop(None)
+        for section, options in sections.items():
+            yield section.upper(), options
+
+    def options_and_values(self, options=None):
+        if options is None:
+            options = self.options
+        for optname, optdict in options:
+            yield (optname, optdict, self.option_value(optname))
+
+
+class ConfigurationMixIn(OptionsManagerMixIn, OptionsProviderMixIn):
+    """basic mixin for simple configurations which don't need the
+    manager / providers model
+    """
+    def __init__(self, *args, **kwargs):
+        if not args:
+            kwargs.setdefault('usage', '')
+        kwargs.setdefault('quiet', 1)
+        OptionsManagerMixIn.__init__(self, *args, **kwargs)
+        OptionsProviderMixIn.__init__(self)
+        if not getattr(self, 'option_groups', None):
+            self.option_groups = []
+            for option, optdict in self.options:
+                try:
+                    gdef = (optdict['group'].upper(), '')
+                except KeyError:
+                    continue
+                if not gdef in self.option_groups:
+                    self.option_groups.append(gdef)
+        self.register_options_provider(self, own_group=0)
+
+    def register_options(self, options):
+        """add some options to the configuration"""
+        options_by_group = {}
+        for optname, optdict in options:
+            options_by_group.setdefault(optdict.get('group', self.name.upper()), []).append((optname, optdict))
+        for group, options in options_by_group.items():
+            self.add_option_group(group, None, options, self)
+        self.options += tuple(options)
+
+    def load_defaults(self):
+        OptionsProviderMixIn.load_defaults(self)
+
+    def __iter__(self):
+        return iter(self.config.__dict__.iteritems())
+
+    def __getitem__(self, key):
+        try:
+            return getattr(self.config, self.option_name(key))
+        except (optparse.OptionValueError, AttributeError):
+            raise KeyError(key)
+
+    def __setitem__(self, key, value):
+        self.set_option(key, value)
+
+    def get(self, key, default=None):
+        try:
+            return getattr(self.config, self.option_name(key))
+        except (OptionError, AttributeError):
+            return default
+
+
+class Configuration(ConfigurationMixIn):
+    """class for simple configurations which don't need the
+    manager / providers model and prefer delegation to inheritance
+
+    configuration values are accessible through a dict like interface
+    """
+
+    def __init__(self, config_file=None, options=None, name=None,
+                 usage=None, doc=None, version=None):
+        if options is not None:
+            self.options = options
+        if name is not None:
+            self.name = name
+        if doc is not None:
+            self.__doc__ = doc
+        super(Configuration, self).__init__(config_file=config_file, usage=usage, version=version)
+
+
+class OptionsManager2ConfigurationAdapter(object):
+    """Adapt an option manager to behave like a
+    `logilab.common.configuration.Configuration` instance
+    """
+    def __init__(self, provider):
+        self.config = provider
+
+    def __getattr__(self, key):
+        return getattr(self.config, key)
+
+    def __getitem__(self, key):
+        provider = self.config._all_options[key]
+        try:
+            return getattr(provider.config, provider.option_name(key))
+        except AttributeError:
+            raise KeyError(key)
+
+    def __setitem__(self, key, value):
+        self.config.global_set_option(self.config.option_name(key), value)
+
+    def get(self, key, default=None):
+        provider = self.config._all_options[key]
+        try:
+            return getattr(provider.config, provider.option_name(key))
+        except AttributeError:
+            return default
+
+
+def read_old_config(newconfig, changes, configfile):
+    """initialize newconfig from a deprecated configuration file
+
+    possible changes:
+    * ('renamed', oldname, newname)
+    * ('moved', option, oldgroup, newgroup)
+    * ('typechanged', option, oldtype, newvalue)
+    """
+    # build an index of changes
+    changesindex = {}
+    for action in changes:
+        if action[0] == 'moved':
+            option, oldgroup, newgroup = action[1:]
+            changesindex.setdefault(option, []).append((action[0], oldgroup, newgroup))
+            continue
+        if action[0] == 'renamed':
+            oldname, newname = action[1:]
+            changesindex.setdefault(newname, []).append((action[0], oldname))
+            continue
+        if action[0] == 'typechanged':
+            option, oldtype, newvalue = action[1:]
+            changesindex.setdefault(option, []).append((action[0], oldtype, newvalue))
+            continue
+        if action[1] in ('added', 'removed'):
+            continue # nothing to do here
+        raise Exception('unknown change %s' % action[0])
+    # build a config object able to read the old config
+    options = []
+    for optname, optdef in newconfig.options:
+        for action in changesindex.pop(optname, ()):
+            if action[0] == 'moved':
+                oldgroup, newgroup = action[1:]
+                optdef = optdef.copy()
+                optdef['group'] = oldgroup
+            elif action[0] == 'renamed':
+                optname = action[1]
+            elif action[0] == 'typechanged':
+                oldtype = action[1]
+                optdef = optdef.copy()
+                optdef['type'] = oldtype
+        options.append((optname, optdef))
+    if changesindex:
+        raise Exception('unapplied changes: %s' % changesindex)
+    oldconfig = Configuration(options=options, name=newconfig.name)
+    # read the old config
+    oldconfig.load_file_configuration(configfile)
+    # apply values reverting changes
+    changes.reverse()
+    done = set()
+    for action in changes:
+        if action[0] == 'renamed':
+            oldname, newname = action[1:]
+            newconfig[newname] = oldconfig[oldname]
+            done.add(newname)
+        elif action[0] == 'typechanged':
+            optname, oldtype, newvalue = action[1:]
+            newconfig[optname] = newvalue
+            done.add(optname)
+    for optname, optdef in newconfig.options:
+        if optdef.get('type') and not optname in done:
+            newconfig.set_option(optname, oldconfig[optname], optdict=optdef)
+
+
+def merge_options(options):
+    """preprocess options to remove duplicate"""
+    alloptions = {}
+    options = list(options)
+    for i in range(len(options)-1, -1, -1):
+        optname, optdict = options[i]
+        if optname in alloptions:
+            options.pop(i)
+            alloptions[optname].update(optdict)
+        else:
+            alloptions[optname] = optdict
+    return tuple(options)

pylibs/logilab/common/contexts.py

@@ -0,0 +1,5 @@
+from warnings import warn
+warn('logilab.common.contexts module is deprecated, use logilab.common.shellutils instead',
+     DeprecationWarning, stacklevel=1)
+
+from logilab.common.shellutils import tempfile, pushd

pylibs/logilab/common/corbautils.py

@@ -0,0 +1,117 @@
+# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+#
+# This file is part of logilab-common.
+#
+# logilab-common is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation, either version 2.1 of the License, or (at your option) any
+# later version.
+#
+# logilab-common is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with logilab-common.  If not, see <http://www.gnu.org/licenses/>.
+"""A set of utility function to ease the use of OmniORBpy.
+
+
+
+
+"""
+__docformat__ = "restructuredtext en"
+
+from omniORB import CORBA, PortableServer
+import CosNaming
+
+orb = None
+
+def get_orb():
+    """
+    returns a reference to the ORB.
+    The first call to the method initialized the ORB
+    This method is mainly used internally in the module.
+    """
+
+    global orb
+    if orb is None:
+        import sys
+        orb = CORBA.ORB_init(sys.argv, CORBA.ORB_ID)
+    return orb
+
+def get_root_context():
+    """
+    returns a reference to the NameService object.
+    This method is mainly used internally in the module.
+    """
+
+    orb = get_orb()
+    nss = orb.resolve_initial_references("NameService")
+    rootContext = nss._narrow(CosNaming.NamingContext)
+    assert rootContext is not None, "Failed to narrow root naming context"
+    return rootContext
+
+def register_object_name(object, namepath):
+    """
+    Registers a object in the NamingService.
+    The name path is a list of 2-uples (id,kind) giving the path.
+
+    For instance if the path of an object is [('foo',''),('bar','')],
+    it is possible to get a reference to the object using the URL
+    'corbaname::hostname#foo/bar'.
+    [('logilab','rootmodule'),('chatbot','application'),('chatter','server')]
+    is mapped to
+    'corbaname::hostname#logilab.rootmodule/chatbot.application/chatter.server'
+
+    The get_object_reference() function can be used to resolve such a URL.
+    """
+    context = get_root_context()
+    for id, kind in namepath[:-1]:
+        name = [CosNaming.NameComponent(id, kind)]
+        try:
+            context = context.bind_new_context(name)
+        except CosNaming.NamingContext.AlreadyBound, ex:
+            context = context.resolve(name)._narrow(CosNaming.NamingContext)
+            assert context is not None, \
+                   'test context exists but is not a NamingContext'
+
+    id, kind = namepath[-1]
+    name = [CosNaming.NameComponent(id, kind)]
+    try:
+        context.bind(name, object._this())
+    except CosNaming.NamingContext.AlreadyBound, ex:
+        context.rebind(name, object._this())
+
+def activate_POA():
+    """
+    This methods activates the Portable Object Adapter.
+    You need to call it to enable the reception of messages in your code,
+    on both the client and the server.
+    """
+    orb = get_orb()
+    poa = orb.resolve_initial_references('RootPOA')
+    poaManager = poa._get_the_POAManager()
+    poaManager.activate()
+
+def run_orb():
+    """
+    Enters the ORB mainloop on the server.
+    You should not call this method on the client.
+    """
+    get_orb().run()
+
+def get_object_reference(url):
+    """
+    Resolves a corbaname URL to an object proxy.
+    See register_object_name() for examples URLs
+    """
+    return get_orb().string_to_object(url)
+
+def get_object_string(host, namepath):
+    """given an host name and a name path as described in register_object_name,
+    return a corba string identifier
+    """
+    strname = '/'.join(['.'.join(path_elt) for path_elt in namepath])
+    return 'corbaname::%s#%s' % (host, strname)

pylibs/logilab/common/daemon.py

@@ -0,0 +1,100 @@
+# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+#
+# This file is part of logilab-common.
+#
+# logilab-common is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation, either version 2.1 of the License, or (at your option) any
+# later version.
+#
+# logilab-common is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with logilab-common.  If not, see <http://www.gnu.org/licenses/>.
+"""A daemonize function (for Unices)"""
+
+__docformat__ = "restructuredtext en"
+
+import os
+import errno
+import signal
+import sys
+import time
+import warnings
+
+def setugid(user):
+    """Change process user and group ID
+
+    Argument is a numeric user id or a user name"""
+    try:
+        from pwd import getpwuid
+        passwd = getpwuid(int(user))
+    except ValueError:
+        from pwd import getpwnam
+        passwd = getpwnam(user)
+
+    if hasattr(os, 'initgroups'): # python >= 2.7
+        os.initgroups(passwd.pw_name, passwd.pw_gid)
+    else:
+        import ctypes
+        if ctypes.CDLL(None).initgroups(passwd.pw_name, passwd.pw_gid) < 0:
+            err = ctypes.c_int.in_dll(ctypes.pythonapi,"errno").value
+            raise OSError(err, os.strerror(err), 'initgroups')
+    os.setgid(passwd.pw_gid)
+    os.setuid(passwd.pw_uid)
+    os.environ['HOME'] = passwd.pw_dir
+
+
+def daemonize(pidfile=None, uid=None, umask=077):
+    """daemonize a Unix process. Set paranoid umask by default.
+
+    Return 1 in the original process, 2 in the first fork, and None for the
+    second fork (eg daemon process).
+    """
+    # http://www.faqs.org/faqs/unix-faq/programmer/faq/
+    #
+    # fork so the parent can exit
+    if os.fork():   # launch child and...
+        return 1
+    # disconnect from tty and create a new session
+    os.setsid()
+    # fork again so the parent, (the session group leader), can exit.
+    # as a non-session group leader, we can never regain a controlling
+    # terminal.
+    if os.fork():   # launch child again.
+        return 2
+    # move to the root to avoit mount pb
+    os.chdir('/')
+    # set umask if specified
+    if umask is not None:
+        os.umask(umask)
+    # redirect standard descriptors
+    null = os.open('/dev/null', os.O_RDWR)
+    for i in range(3):
+        try:
+            os.dup2(null, i)
+        except OSError, e:
+            if e.errno != errno.EBADF:
+                raise
+    os.close(null)
+    # filter warnings
+    warnings.filterwarnings('ignore')
+    # write pid in a file
+    if pidfile:
+        # ensure the directory where the pid-file should be set exists (for
+        # instance /var/run/cubicweb may be deleted on computer restart)
+        piddir = os.path.dirname(pidfile)
+        if not os.path.exists(piddir):
+            os.makedirs(piddir)
+        f = file(pidfile, 'w')
+        f.write(str(os.getpid()))
+        f.close()
+        os.chmod(pidfile, 0644)
+    # change process uid
+    if uid:
+        setugid(uid)
+    return None

pylibs/logilab/common/date.py

@@ -0,0 +1,327 @@
+# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+#
+# This file is part of logilab-common.
+#
+# logilab-common is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation, either version 2.1 of the License, or (at your option) any
+# later version.
+#
+# logilab-common is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with logilab-common.  If not, see <http://www.gnu.org/licenses/>.
+"""Date manipulation helper functions."""
+from __future__ import division
+
+__docformat__ = "restructuredtext en"
+
+import math
+import re
+from locale import getpreferredencoding
+from datetime import date, time, datetime, timedelta
+from time import strptime as time_strptime
+from calendar import monthrange, timegm
+
+try:
+    from mx.DateTime import RelativeDateTime, Date, DateTimeType
+except ImportError:
+    endOfMonth = None
+    DateTimeType = datetime
+else:
+    endOfMonth = RelativeDateTime(months=1, day=-1)
+
+# NOTE: should we implement a compatibility layer between date representations
+#       as we have in lgc.db ?
+
+FRENCH_FIXED_HOLIDAYS = {
+    'jour_an': '%s-01-01',
+    'fete_travail': '%s-05-01',
+    'armistice1945': '%s-05-08',
+    'fete_nat': '%s-07-14',
+    'assomption': '%s-08-15',
+    'toussaint': '%s-11-01',
+    'armistice1918': '%s-11-11',
+    'noel': '%s-12-25',
+    }
+
+FRENCH_MOBILE_HOLIDAYS = {
+    'paques2004': '2004-04-12',
+    'ascension2004': '2004-05-20',
+    'pentecote2004': '2004-05-31',
+
+    'paques2005': '2005-03-28',
+    'ascension2005': '2005-05-05',
+    'pentecote2005': '2005-05-16',
+
+    'paques2006': '2006-04-17',
+    'ascension2006': '2006-05-25',
+    'pentecote2006': '2006-06-05',
+
+    'paques2007': '2007-04-09',
+    'ascension2007': '2007-05-17',
+    'pentecote2007': '2007-05-28',
+
+    'paques2008': '2008-03-24',
+    'ascension2008': '2008-05-01',
+    'pentecote2008': '2008-05-12',
+
+    'paques2009': '2009-04-13',
+    'ascension2009': '2009-05-21',
+    'pentecote2009': '2009-06-01',
+
+    'paques2010': '2010-04-05',
+    'ascension2010': '2010-05-13',
+    'pentecote2010': '2010-05-24',
+
+    'paques2011': '2011-04-25',
+    'ascension2011': '2011-06-02',
+    'pentecote2011': '2011-06-13',
+
+    'paques2012': '2012-04-09',
+    'ascension2012': '2012-05-17',
+    'pentecote2012': '2012-05-28',
+    }
+
+# XXX this implementation cries for multimethod dispatching
+
+def get_step(dateobj, nbdays=1):
+    # assume date is either a python datetime or a mx.DateTime object
+    if isinstance(dateobj, date):
+        return ONEDAY * nbdays
+    return nbdays # mx.DateTime is ok with integers
+
+def datefactory(year, month, day, sampledate):
+    # assume date is either a python datetime or a mx.DateTime object
+    if isinstance(sampledate, datetime):
+        return datetime(year, month, day)
+    if isinstance(sampledate, date):
+        return date(year, month, day)
+    return Date(year, month, day)
+
+def weekday(dateobj):
+    # assume date is either a python datetime or a mx.DateTime object
+    if isinstance(dateobj, date):
+        return dateobj.weekday()
+    return dateobj.day_of_week
+
+def str2date(datestr, sampledate):
+    # NOTE: datetime.strptime is not an option until we drop py2.4 compat
+    year, month, day = [int(chunk) for chunk in datestr.split('-')]
+    return datefactory(year, month, day, sampledate)
+
+def days_between(start, end):
+    if isinstance(start, date):
+        delta = end - start
+        # datetime.timedelta.days is always an integer (floored)
+        if delta.seconds:
+            return delta.days + 1
+        return delta.days
+    else:
+        return int(math.ceil((end - start).days))
+
+def get_national_holidays(begin, end):
+    """return french national days off between begin and end"""
+    begin = datefactory(begin.year, begin.month, begin.day, begin)
+    end = datefactory(end.year, end.month, end.day, end)
+    holidays = [str2date(datestr, begin)
+                for datestr in FRENCH_MOBILE_HOLIDAYS.values()]
+    for year in xrange(begin.year, end.year+1):
+        for datestr in FRENCH_FIXED_HOLIDAYS.values():
+            date = str2date(datestr % year, begin)
+            if date not in holidays:
+                holidays.append(date)
+    return [day for day in holidays if begin <= day < end]
+
+def add_days_worked(start, days):
+    """adds date but try to only take days worked into account"""
+    step = get_step(start)
+    weeks, plus = divmod(days, 5)
+    end = start + ((weeks * 7) + plus) * step
+    if weekday(end) >= 5: # saturday or sunday
+        end += (2 * step)
+    end += len([x for x in get_national_holidays(start, end + step)
+                if weekday(x) < 5]) * step
+    if weekday(end) >= 5: # saturday or sunday
+        end += (2 * step)
+    return end
+
+def nb_open_days(start, end):
+    assert start <= end
+    step = get_step(start)
+    days = days_between(start, end)
+    weeks, plus = divmod(days, 7)
+    if weekday(start) > weekday(end):
+        plus -= 2
+    elif weekday(end) == 6:
+        plus -= 1
+    open_days = weeks * 5 + plus
+    nb_week_holidays = len([x for x in get_national_holidays(start, end+step)
+                            if weekday(x) < 5 and x < end])
+    open_days -= nb_week_holidays
+    if open_days < 0:
+        return 0
+    return open_days
+
+def date_range(begin, end, incday=None, incmonth=None):
+    """yields each date between begin and end
+
+    :param begin: the start date
+    :param end: the end date
+    :param incr: the step to use to iterate over dates. Default is
+                 one day.
+    :param include: None (means no exclusion) or a function taking a
+                    date as parameter, and returning True if the date
+                    should be included.
+
+    When using mx datetime, you should *NOT* use incmonth argument, use instead
+    oneDay, oneHour, oneMinute, oneSecond, oneWeek or endOfMonth (to enumerate
+    months) as `incday` argument
+    """
+    assert not (incday and incmonth)
+    begin = todate(begin)
+    end = todate(end)
+    if incmonth:
+        while begin < end:
+            begin = next_month(begin, incmonth)
+            yield begin
+    else:
+        incr = get_step(begin, incday or 1)
+        while begin < end:
+           yield begin
+           begin += incr
+
+# makes py datetime usable #####################################################
+
+ONEDAY = timedelta(days=1)
+ONEWEEK = timedelta(days=7)
+
+try:
+    strptime = datetime.strptime
+except AttributeError: # py < 2.5
+    from time import strptime as time_strptime
+    def strptime(value, format):
+        return datetime(*time_strptime(value, format)[:6])
+
+def strptime_time(value, format='%H:%M'):
+    return time(*time_strptime(value, format)[3:6])
+
+def todate(somedate):
+    """return a date from a date (leaving unchanged) or a datetime"""
+    if isinstance(somedate, datetime):
+        return date(somedate.year, somedate.month, somedate.day)
+    assert isinstance(somedate, (date, DateTimeType)), repr(somedate)
+    return somedate
+
+def totime(somedate):
+    """return a time from a time (leaving unchanged), date or datetime"""
+    # XXX mx compat
+    if not isinstance(somedate, time):
+        return time(somedate.hour, somedate.minute, somedate.second)
+    assert isinstance(somedate, (time)), repr(somedate)
+    return somedate
+
+def todatetime(somedate):
+    """return a date from a date (leaving unchanged) or a datetime"""
+    # take care, datetime is a subclass of date
+    if isinstance(somedate, datetime):
+        return somedate
+    assert isinstance(somedate, (date, DateTimeType)), repr(somedate)
+    return datetime(somedate.year, somedate.month, somedate.day)
+
+def datetime2ticks(somedate):
+    return timegm(somedate.timetuple()) * 1000
+
+def ticks2datetime(ticks):
+    miliseconds, microseconds = divmod(ticks, 1000)
+    try:
+        return datetime.fromtimestamp(miliseconds)
+    except (ValueError, OverflowError):
+        epoch = datetime.fromtimestamp(0)
+        nb_days, seconds = divmod(int(miliseconds), 86400)
+        delta = timedelta(nb_days, seconds=seconds, microseconds=microseconds)
+        try:
+            return epoch + delta
+        except (ValueError, OverflowError):
+            raise
+
+def days_in_month(somedate):
+    return monthrange(somedate.year, somedate.month)[1]
+
+def days_in_year(somedate):
+    feb = date(somedate.year, 2, 1)
+    if days_in_month(feb) == 29:
+        return 366
+    else:
+        return 365
+
+def previous_month(somedate, nbmonth=1):
+    while nbmonth:
+        somedate = first_day(somedate) - ONEDAY
+        nbmonth -= 1
+    return somedate
+
+def next_month(somedate, nbmonth=1):
+    while nbmonth:
+        somedate = last_day(somedate) + ONEDAY
+        nbmonth -= 1
+    return somedate
+
+def first_day(somedate):
+    return date(somedate.year, somedate.month, 1)
+
+def last_day(somedate):
+    return date(somedate.year, somedate.month, days_in_month(somedate))
+
+def ustrftime(somedate, fmt='%Y-%m-%d'):
+    """like strftime, but returns a unicode string instead of an encoded
+    string which' may be problematic with localized date.
+
+    encoding is guessed by locale.getpreferredencoding()
+    """
+    encoding = getpreferredencoding(do_setlocale=False) or 'UTF-8'
+    try:
+        return unicode(somedate.strftime(str(fmt)), encoding)
+    except ValueError, exc:
+        if somedate.year >= 1900:
+            raise
+        # datetime is not happy with dates before 1900
+        # we try to work around this, assuming a simple
+        # format string
+        fields = {'Y': somedate.year,
+                  'm': somedate.month,
+                  'd': somedate.day,
+                  }
+        if isinstance(somedate, datetime):
+            fields.update({'H': somedate.hour,
+                           'M': somedate.minute,
+                           'S': somedate.second})
+        fmt = re.sub('%([YmdHMS])', r'%(\1)02d', fmt)
+        return unicode(fmt) % fields
+
+def utcdatetime(dt):
+    if dt.tzinfo is None:
+        return dt
+    return datetime(*dt.utctimetuple()[:7])
+
+def utctime(dt):
+    if dt.tzinfo is None:
+        return dt
+    return (dt + dt.utcoffset() + dt.dst()).replace(tzinfo=None)
+
+def datetime_to_seconds(date):
+    """return the number of seconds since the begining of the day for that date
+    """
+    return date.second+60*date.minute + 3600*date.hour
+
+def timedelta_to_days(delta):
+    """return the time delta as a number of seconds"""
+    return delta.days + delta.seconds / (3600*24)
+
+def timedelta_to_seconds(delta):
+    """return the time delta as a fraction of days"""
+    return delta.days*(3600*24) + delta.seconds

pylibs/logilab/common/dbf.py

@@ -0,0 +1,229 @@
+# -*- coding: utf-8 -*-
+# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+#
+# This file is part of logilab-common.
+#
+# logilab-common is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation, either version 2.1 of the License, or (at your option) any
+# later version.
+#
+# logilab-common is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with logilab-common.  If not, see <http://www.gnu.org/licenses/>.
+"""This is a DBF reader which reads Visual Fox Pro DBF format with Memo field
+
+Usage:
+
+>>> rec = readDbf('test.dbf')
+>>> for line in rec:
+>>>     print line['name']
+
+
+:date: 13/07/2007
+
+http://www.physics.ox.ac.uk/users/santoso/Software.Repository.html
+page says code is "available as is without any warranty or support".
+"""
+
+import struct
+import os, os.path
+import sys
+import csv
+import tempfile
+import ConfigParser
+
+class Dbase:
+    def __init__(self):
+        self.fdb = None
+        self.fmemo = None
+        self.db_data = None
+        self.memo_data = None
+        self.fields = None
+        self.num_records = 0
+        self.header = None
+        self.memo_file = ''
+        self.memo_header = None
+        self.memo_block_size = 0
+        self.memo_header_len = 0
+
+    def _drop_after_NULL(self, txt):
+        for i in range(0, len(txt)):
+            if ord(struct.unpack('c', txt[i])[0])==0:
+                return txt[:i]
+        return txt
+
+    def _reverse_endian(self, num):
+        if not len(num):
+            return 0
+        val = struct.unpack('<L', num)
+        val = struct.pack('>L', val[0])
+        val = struct.unpack('>L', val)
+        return val[0]
+
+    def _assign_ids(self, lst, ids):
+        result = {}
+        idx = 0
+        for item in lst:
+            id = ids[idx]
+            result[id] = item
+            idx += 1
+        return result
+
+    def open(self, db_name):
+        filesize = os.path.getsize(db_name)
+        if filesize <= 68:
+            raise IOError, 'The file is not large enough to be a dbf file'
+
+        self.fdb = open(db_name, 'rb')
+
+        self.memo_file = ''
+        if os.path.isfile(db_name[0:-1] + 't'):
+            self.memo_file = db_name[0:-1] + 't'
+        elif os.path.isfile(db_name[0:-3] + 'fpt'):
+            self.memo_file = db_name[0:-3] + 'fpt'
+
+        if self.memo_file:
+            #Read memo file
+            self.fmemo = open(self.memo_file, 'rb')
+            self.memo_data = self.fmemo.read()
+            self.memo_header = self._assign_ids(struct.unpack('>6x1H', self.memo_data[:8]), ['Block size'])
+            block_size = self.memo_header['Block size']
+            if not block_size:
+                block_size = 512
+            self.memo_block_size = block_size
+            self.memo_header_len = block_size
+            memo_size = os.path.getsize(self.memo_file)
+
+        #Start reading data file
+        data = self.fdb.read(32)
+        self.header = self._assign_ids(struct.unpack('<B 3B L 2H 20x', data), ['id', 'Year', 'Month', 'Day', '# of Records', 'Header Size', 'Record Size'])
+        self.header['id'] = hex(self.header['id'])
+
+        self.num_records = self.header['# of Records']
+        data = self.fdb.read(self.header['Header Size']-34)
+        self.fields = {}
+        x = 0
+        header_pattern = '<11s c 4x B B 14x'
+        ids = ['Field Name', 'Field Type', 'Field Length', 'Field Precision']
+        pattern_len = 32
+        for offset in range(0, len(data), 32):
+            if ord(data[offset])==0x0d:
+                break
+            x += 1
+            data_subset = data[offset: offset+pattern_len]
+            if len(data_subset) < pattern_len:
+                data_subset += ' '*(pattern_len-len(data_subset))
+            self.fields[x] = self._assign_ids(struct.unpack(header_pattern, data_subset), ids)
+            self.fields[x]['Field Name'] = self._drop_after_NULL(self.fields[x]['Field Name'])
+
+        self.fdb.read(3)
+        if self.header['# of Records']:
+            data_size = (self.header['# of Records'] * self.header['Record Size']) - 1
+            self.db_data = self.fdb.read(data_size)
+        else:
+            self.db_data = ''
+        self.row_format = '<'
+        self.row_ids = []
+        self.row_len = 0
+        for key in self.fields:
+            field = self.fields[key]
+            self.row_format += '%ds ' % (field['Field Length'])
+            self.row_ids.append(field['Field Name'])
+            self.row_len += field['Field Length']
+
+    def close(self):
+        if self.fdb:
+            self.fdb.close()
+        if self.fmemo:
+            self.fmemo.close()
+
+    def get_numrecords(self):
+        return self.num_records
+
+    def get_record_with_names(self, rec_no):
+        """
+        This function accept record number from 0 to N-1
+        """
+        if rec_no < 0 or rec_no > self.num_records:
+            raise Exception, 'Unable to extract data outside the range'
+
+        offset = self.header['Record Size'] * rec_no
+        data = self.db_data[offset:offset+self.row_len]
+        record = self._assign_ids(struct.unpack(self.row_format, data), self.row_ids)
+
+        if self.memo_file:
+            for key in self.fields:
+                field = self.fields[key]
+                f_type = field['Field Type']
+                f_name = field['Field Name']
+                c_data = record[f_name]
+
+                if f_type=='M' or f_type=='G' or f_type=='B' or f_type=='P':
+                    c_data = self._reverse_endian(c_data)
+                    if c_data:
+                        record[f_name] = self.read_memo(c_data-1).strip()
+                else:
+                    record[f_name] = c_data.strip()
+        return record
+
+    def read_memo_record(self, num, in_length):
+        """
+        Read the record of given number. The second parameter is the length of
+        the record to read. It can be undefined, meaning read the whole record,
+        and it can be negative, meaning at most the length
+        """
+        if in_length < 0:
+            in_length = -self.memo_block_size
+
+        offset = self.memo_header_len + num * self.memo_block_size
+        self.fmemo.seek(offset)
+        if in_length<0:
+            in_length = -in_length
+        if in_length==0:
+            return ''
+        return self.fmemo.read(in_length)
+
+    def read_memo(self, num):
+        result = ''
+        buffer = self.read_memo_record(num, -1)
+        if len(buffer)<=0:
+            return ''
+        length = struct.unpack('>L', buffer[4:4+4])[0] + 8
+
+        block_size = self.memo_block_size
+        if length < block_size:
+            return buffer[8:length]
+        rest_length = length - block_size
+        rest_data = self.read_memo_record(num+1, rest_length)
+        if len(rest_data)<=0:
+            return ''
+        return buffer[8:] + rest_data
+
+def readDbf(filename):
+    """
+    Read the DBF file specified by the filename and
+    return the records as a list of dictionary.
+
+    :param: filename File name of the DBF
+    :return: List of rows
+    """
+    db = Dbase()
+    db.open(filename)
+    num = db.get_numrecords()
+    rec = []
+    for i in range(0, num):
+        record = db.get_record_with_names(i)
+        rec.append(record)
+    db.close()
+    return  rec
+
+if __name__=='__main__':
+    rec = readDbf('dbf/sptable.dbf')
+    for line in rec:
+        print '%s %s' % (line['GENUS'].strip(), line['SPECIES'].strip())

pylibs/logilab/common/debugger.py

@@ -0,0 +1,210 @@
+# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+#
+# This file is part of logilab-common.
+#
+# logilab-common is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation, either version 2.1 of the License, or (at your option) any
+# later version.
+#
+# logilab-common is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with logilab-common.  If not, see <http://www.gnu.org/licenses/>.
+"""Customized version of pdb's default debugger.
+
+- sets up a history file
+- uses ipython if available to colorize lines of code
+- overrides list command to search for current block instead
+  of using 5 lines of context
+
+
+
+
+"""
+__docformat__ = "restructuredtext en"
+
+try:
+    import readline
+except ImportError:
+    readline = None
+import os
+import os.path as osp
+import sys
+from pdb import Pdb
+from cStringIO import StringIO
+import inspect
+
+try:
+    from IPython import PyColorize
+except ImportError:
+    def colorize(source, *args):
+        """fallback colorize function"""
+        return source
+    def colorize_source(source, *args):
+        return source
+else:
+    def colorize(source, start_lineno, curlineno):
+        """colorize and annotate source with linenos
+        (as in pdb's list command)
+        """
+        parser = PyColorize.Parser()
+        output = StringIO()
+        parser.format(source, output)
+        annotated = []
+        for index, line in enumerate(output.getvalue().splitlines()):
+            lineno = index + start_lineno
+            if lineno == curlineno:
+                annotated.append('%4s\t->\t%s' % (lineno, line))
+            else:
+                annotated.append('%4s\t\t%s' % (lineno, line))
+        return '\n'.join(annotated)
+
+    def colorize_source(source):
+        """colorize given source"""
+        parser = PyColorize.Parser()
+        output = StringIO()
+        parser.format(source, output)
+        return output.getvalue()
+
+
+def getsource(obj):
+    """Return the text of the source code for an object.
+
+    The argument may be a module, class, method, function, traceback, frame,
+    or code object.  The source code is returned as a single string.  An
+    IOError is raised if the source code cannot be retrieved."""
+    lines, lnum = inspect.getsourcelines(obj)
+    return ''.join(lines), lnum
+
+
+################################################################
+class Debugger(Pdb):
+    """custom debugger
+
+    - sets up a history file
+    - uses ipython if available to colorize lines of code
+    - overrides list command to search for current block instead
+      of using 5 lines of context
+    """
+    def __init__(self, tcbk=None):
+        Pdb.__init__(self)
+        self.reset()
+        if tcbk:
+            while tcbk.tb_next is not None:
+                tcbk = tcbk.tb_next
+        self._tcbk = tcbk
+        self._histfile = os.path.expanduser("~/.pdbhist")
+
+    def setup_history_file(self):
+        """if readline is available, read pdb history file
+        """
+        if readline is not None:
+            try:
+                # XXX try..except shouldn't be necessary
+                # read_history_file() can accept None
+                readline.read_history_file(self._histfile)
+            except IOError:
+                pass
+
+    def start(self):
+        """starts the interactive mode"""
+        self.interaction(self._tcbk.tb_frame, self._tcbk)
+
+    def setup(self, frame, tcbk):
+        """setup hook: set up history file"""
+        self.setup_history_file()
+        Pdb.setup(self, frame, tcbk)
+
+    def set_quit(self):
+        """quit hook: save commands in the history file"""
+        if readline is not None:
+            readline.write_history_file(self._histfile)
+        Pdb.set_quit(self)
+
+    def complete_p(self, text, line, begin_idx, end_idx):
+        """provide variable names completion for the ``p`` command"""
+        namespace = dict(self.curframe.f_globals)
+        namespace.update(self.curframe.f_locals)
+        if '.' in text:
+            return self.attr_matches(text, namespace)
+        return [varname for varname in namespace if varname.startswith(text)]
+
+
+    def attr_matches(self, text, namespace):
+        """implementation coming from rlcompleter.Completer.attr_matches
+        Compute matches when text contains a dot.
+
+        Assuming the text is of the form NAME.NAME....[NAME], and is
+        evaluatable in self.namespace, it will be evaluated and its attributes
+        (as revealed by dir()) are used as possible completions.  (For class
+        instances, class members are also considered.)
+
+        WARNING: this can still invoke arbitrary C code, if an object
+        with a __getattr__ hook is evaluated.
+
+        """
+        import re
+        m = re.match(r"(\w+(\.\w+)*)\.(\w*)", text)
+        if not m:
+            return
+        expr, attr = m.group(1, 3)
+        object = eval(expr, namespace)
+        words = dir(object)
+        if hasattr(object, '__class__'):
+            words.append('__class__')
+            words = words + self.get_class_members(object.__class__)
+        matches = []
+        n = len(attr)
+        for word in words:
+            if word[:n] == attr and word != "__builtins__":
+                matches.append("%s.%s" % (expr, word))
+        return matches
+
+    def get_class_members(self, klass):
+        """implementation coming from rlcompleter.get_class_members"""
+        ret = dir(klass)
+        if hasattr(klass, '__bases__'):
+            for base in klass.__bases__:
+                ret = ret + self.get_class_members(base)
+        return ret
+
+    ## specific / overridden commands
+    def do_list(self, arg):
+        """overrides default list command to display the surrounding block
+        instead of 5 lines of context
+        """
+        self.lastcmd = 'list'
+        if not arg:
+            try:
+                source, start_lineno = getsource(self.curframe)
+                print colorize(''.join(source), start_lineno,
+                               self.curframe.f_lineno)
+            except KeyboardInterrupt:
+                pass
+            except IOError:
+                Pdb.do_list(self, arg)
+        else:
+            Pdb.do_list(self, arg)
+    do_l = do_list
+
+    def do_open(self, arg):
+        """opens source file corresponding to the current stack level"""
+        filename = self.curframe.f_code.co_filename
+        lineno = self.curframe.f_lineno
+        cmd = 'emacsclient --no-wait +%s %s' % (lineno, filename)
+        os.system(cmd)
+
+    do_o = do_open
+
+def pm():
+    """use our custom debugger"""
+    dbg = Debugger(sys.last_traceback)
+    dbg.start()
+
+def set_trace():
+    Debugger().set_trace(sys._getframe().f_back)

pylibs/logilab/common/decorators.py

@@ -0,0 +1,283 @@
+# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+#
+# This file is part of logilab-common.
+#
+# logilab-common is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation, either version 2.1 of the License, or (at your option) any
+# later version.
+#
+# logilab-common is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with logilab-common.  If not, see <http://www.gnu.org/licenses/>.
+""" A few useful function/method decorators. """
+__docformat__ = "restructuredtext en"
+
+import sys
+from time import clock, time
+
+from logilab.common.compat import callable, method_type
+
+# XXX rewrite so we can use the decorator syntax when keyarg has to be specified
+
+def _is_generator_function(callableobj):
+    return callableobj.func_code.co_flags & 0x20
+
+class cached_decorator(object):
+    def __init__(self, cacheattr=None, keyarg=None):
+        self.cacheattr = cacheattr
+        self.keyarg = keyarg
+    def __call__(self, callableobj=None):
+        assert not _is_generator_function(callableobj), \
+               'cannot cache generator function: %s' % callableobj
+        if callableobj.func_code.co_argcount == 1 or self.keyarg == 0:
+            cache = _SingleValueCache(callableobj, self.cacheattr)
+        elif self.keyarg:
+            cache = _MultiValuesKeyArgCache(callableobj, self.keyarg, self.cacheattr)
+        else:
+            cache = _MultiValuesCache(callableobj, self.cacheattr)
+        return cache.closure()
+
+class _SingleValueCache(object):
+    def __init__(self, callableobj, cacheattr=None):
+        self.callable = callableobj
+        if cacheattr is None:
+            self.cacheattr = '_%s_cache_' % callableobj.__name__
+        else:
+            assert cacheattr != callableobj.__name__
+            self.cacheattr = cacheattr
+
+    def __call__(__me, self, *args):
+        try:
+            return self.__dict__[__me.cacheattr]
+        except KeyError:
+            value = __me.callable(self, *args)
+            setattr(self, __me.cacheattr, value)
+            return value
+
+    def closure(self):
+        def wrapped(*args, **kwargs):
+            return self.__call__(*args, **kwargs)
+        wrapped.cache_obj = self
+        try:
+            wrapped.__doc__ = self.callable.__doc__
+            wrapped.__name__ = self.callable.__name__
+            wrapped.func_name = self.callable.func_name
+        except:
+            pass
+        return wrapped
+
+    def clear(self, holder):
+        holder.__dict__.pop(self.cacheattr, None)
+
+
+class _MultiValuesCache(_SingleValueCache):
+    def _get_cache(self, holder):
+        try:
+            _cache = holder.__dict__[self.cacheattr]
+        except KeyError:
+            _cache = {}
+            setattr(holder, self.cacheattr, _cache)
+        return _cache
+
+    def __call__(__me, self, *args, **kwargs):
+        _cache = __me._get_cache(self)
+        try:
+            return _cache[args]
+        except KeyError:
+            _cache[args] = __me.callable(self, *args)
+            return _cache[args]
+
+class _MultiValuesKeyArgCache(_MultiValuesCache):
+    def __init__(self, callableobj, keyarg, cacheattr=None):
+        super(_MultiValuesKeyArgCache, self).__init__(callableobj, cacheattr)
+        self.keyarg = keyarg
+
+    def __call__(__me, self, *args, **kwargs):
+        _cache = __me._get_cache(self)
+        key = args[__me.keyarg-1]
+        try:
+            return _cache[key]
+        except KeyError:
+            _cache[key] = __me.callable(self, *args, **kwargs)
+            return _cache[key]
+
+
+def cached(callableobj=None, keyarg=None, **kwargs):
+    """Simple decorator to cache result of method call."""
+    kwargs['keyarg'] = keyarg
+    decorator = cached_decorator(**kwargs)
+    if callableobj is None:
+        return decorator
+    else:
+        return decorator(callableobj)
+
+
+class cachedproperty(object):
+    """ Provides a cached property equivalent to the stacking of
+    @cached and @property, but more efficient.
+
+    After first usage, the <property_name> becomes part of the object's
+    __dict__. Doing:
+
+      del obj.<property_name> empties the cache.
+
+    Idea taken from the pyramid_ framework and the mercurial_ project.
+
+    .. _pyramid: http://pypi.python.org/pypi/pyramid
+    .. _mercurial: http://pypi.python.org/pypi/Mercurial
+    """
+    __slots__ = ('wrapped',)
+
+    def __init__(self, wrapped):
+        try:
+            wrapped.__name__
+        except AttributeError:
+            raise TypeError('%s must have a __name__ attribute' %
+                            wrapped)
+        self.wrapped = wrapped
+
+    @property
+    def __doc__(self):
+        doc = getattr(self.wrapped, '__doc__', None)
+        return ('<wrapped by the cachedproperty decorator>%s'
+                % ('\n%s' % doc if doc else ''))
+
+    def __get__(self, inst, objtype=None):
+        if inst is None:
+            return self
+        val = self.wrapped(inst)
+        setattr(inst, self.wrapped.__name__, val)
+        return val
+
+
+def get_cache_impl(obj, funcname):
+    cls = obj.__class__
+    member = getattr(cls, funcname)
+    if isinstance(member, property):
+        member = member.fget
+    return member.cache_obj
+
+def clear_cache(obj, funcname):
+    """Clear a cache handled by the :func:`cached` decorator. If 'x' class has
+    @cached on its method `foo`, type
+
+    >>> clear_cache(x, 'foo')
+
+    to purge this method's cache on the instance.
+    """
+    get_cache_impl(obj, funcname).clear(obj)
+
+def copy_cache(obj, funcname, cacheobj):
+    """Copy cache for <funcname> from cacheobj to obj."""
+    cacheattr = get_cache_impl(obj, funcname).cacheattr
+    try:
+        setattr(obj, cacheattr, cacheobj.__dict__[cacheattr])
+    except KeyError:
+        pass
+
+
+class wproperty(object):
+    """Simple descriptor expecting to take a modifier function as first argument
+    and looking for a _<function name> to retrieve the attribute.
+    """
+    def __init__(self, setfunc):
+        self.setfunc = setfunc
+        self.attrname = '_%s' % setfunc.__name__
+
+    def __set__(self, obj, value):
+        self.setfunc(obj, value)
+
+    def __get__(self, obj, cls):
+        assert obj is not None
+        return getattr(obj, self.attrname)
+
+
+class classproperty(object):
+    """this is a simple property-like class but for class attributes.
+    """
+    def __init__(self, get):
+        self.get = get
+    def __get__(self, inst, cls):
+        return self.get(cls)
+
+
+class iclassmethod(object):
+    '''Descriptor for method which should be available as class method if called
+    on the class or instance method if called on an instance.
+    '''
+    def __init__(self, func):
+        self.func = func
+    def __get__(self, instance, objtype):
+        if instance is None:
+            return method_type(self.func, objtype, objtype.__class__)
+        return method_type(self.func, instance, objtype)
+    def __set__(self, instance, value):
+        raise AttributeError("can't set attribute")
+
+
+def timed(f):
+    def wrap(*args, **kwargs):
+        t = time()
+        c = clock()
+        res = f(*args, **kwargs)
+        print '%s clock: %.9f / time: %.9f' % (f.__name__,
+                                               clock() - c, time() - t)
+        return res
+    return wrap
+
+
+def locked(acquire, release):
+    """Decorator taking two methods to acquire/release a lock as argument,
+    returning a decorator function which will call the inner method after
+    having called acquire(self) et will call release(self) afterwards.
+    """
+    def decorator(f):
+        def wrapper(self, *args, **kwargs):
+            acquire(self)
+            try:
+                return f(self, *args, **kwargs)
+            finally:
+                release(self)
+        return wrapper
+    return decorator
+
+
+def monkeypatch(klass, methodname=None):
+    """Decorator extending class with the decorated callable
+    >>> class A:
+    ...     pass
+    >>> @monkeypatch(A)
+    ... def meth(self):
+    ...     return 12
+    ...
+    >>> a = A()
+    >>> a.meth()
+    12
+    >>> @monkeypatch(A, 'foo')
+    ... def meth(self):
+    ...     return 12
+    ...
+    >>> a.foo()
+    12
+    """
+    def decorator(func):
+        try:
+            name = methodname or func.__name__
+        except AttributeError:
+            raise AttributeError('%s has no __name__ attribute: '
+                                 'you should provide an explicit `methodname`'
+                                 % func)
+        if callable(func) and sys.version_info < (3, 0):
+            setattr(klass, name, method_type(func, None, klass))
+        else:
+            # likely a property
+            # this is quite borderline but usage already in the wild ...
+            setattr(klass, name, func)
+        return func
+    return decorator

pylibs/logilab/common/deprecation.py

@@ -0,0 +1,130 @@
+# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+#
+# This file is part of logilab-common.
+#
+# logilab-common is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation, either version 2.1 of the License, or (at your option) any
+# later version.
+#
+# logilab-common is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with logilab-common.  If not, see <http://www.gnu.org/licenses/>.
+"""Deprecation utilities."""
+
+__docformat__ = "restructuredtext en"
+
+import sys
+from warnings import warn
+
+class class_deprecated(type):
+    """metaclass to print a warning on instantiation of a deprecated class"""
+
+    def __call__(cls, *args, **kwargs):
+        msg = getattr(cls, "__deprecation_warning__",
+                      "%(cls)s is deprecated") % {'cls': cls.__name__}
+        warn(msg, DeprecationWarning, stacklevel=2)
+        return type.__call__(cls, *args, **kwargs)
+
+
+def class_renamed(old_name, new_class, message=None):
+    """automatically creates a class which fires a DeprecationWarning
+    when instantiated.
+
+    >>> Set = class_renamed('Set', set, 'Set is now replaced by set')
+    >>> s = Set()
+    sample.py:57: DeprecationWarning: Set is now replaced by set
+      s = Set()
+    >>>
+    """
+    clsdict = {}
+    if message is None:
+        message = '%s is deprecated, use %s' % (old_name, new_class.__name__)
+    clsdict['__deprecation_warning__'] = message
+    try:
+        # new-style class
+        return class_deprecated(old_name, (new_class,), clsdict)
+    except (NameError, TypeError):
+        # old-style class
+        class DeprecatedClass(new_class):
+            """FIXME: There might be a better way to handle old/new-style class
+            """
+            def __init__(self, *args, **kwargs):
+                warn(message, DeprecationWarning, stacklevel=2)
+                new_class.__init__(self, *args, **kwargs)
+        return DeprecatedClass
+
+
+def class_moved(new_class, old_name=None, message=None):
+    """nice wrapper around class_renamed when a class has been moved into
+    another module
+    """
+    if old_name is None:
+        old_name = new_class.__name__
+    if message is None:
+        message = 'class %s is now available as %s.%s' % (
+            old_name, new_class.__module__, new_class.__name__)
+    return class_renamed(old_name, new_class, message)
+
+def deprecated(reason=None, stacklevel=2, name=None, doc=None):
+    """Decorator that raises a DeprecationWarning to print a message
+    when the decorated function is called.
+    """
+    def deprecated_decorator(func):
+        message = reason or 'The function "%s" is deprecated'
+        if '%s' in message:
+            message = message % func.func_name
+        def wrapped(*args, **kwargs):
+            warn(message, DeprecationWarning, stacklevel=stacklevel)
+            return func(*args, **kwargs)
+        try:
+            wrapped.__name__ = name or func.__name__
+        except TypeError: # readonly attribute in 2.3
+            pass
+        wrapped.__doc__ = doc or func.__doc__
+        return wrapped
+    return deprecated_decorator
+
+def moved(modpath, objname):
+    """use to tell that a callable has been moved to a new module.
+
+    It returns a callable wrapper, so that when its called a warning is printed
+    telling where the object can be found, import is done (and not before) and
+    the actual object is called.
+
+    NOTE: the usage is somewhat limited on classes since it will fail if the
+    wrapper is use in a class ancestors list, use the `class_moved` function
+    instead (which has no lazy import feature though).
+    """
+    def callnew(*args, **kwargs):
+        from logilab.common.modutils import load_module_from_name
+        message = "object %s has been moved to module %s" % (objname, modpath)
+        warn(message, DeprecationWarning, stacklevel=2)
+        m = load_module_from_name(modpath)
+        return getattr(m, objname)(*args, **kwargs)
+    return callnew
+
+
+
+class DeprecationWrapper(object):
+    """proxy to print a warning on access to any attribute of the wrapped object
+    """
+    def __init__(self, proxied, msg=None):
+        self._proxied = proxied
+        self._msg = msg
+
+    def __getattr__(self, attr):
+        warn(self._msg, DeprecationWarning, stacklevel=2)
+        return getattr(self._proxied, attr)
+
+    def __setattr__(self, attr, value):
+        if attr in ('_proxied', '_msg'):
+            self.__dict__[attr] = value
+        else:
+            warn(self._msg, DeprecationWarning, stacklevel=2)
+            setattr(self._proxied, attr, value)

pylibs/logilab/common/fileutils.py

@@ -0,0 +1,402 @@
+# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+#
+# This file is part of logilab-common.
+#
+# logilab-common is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation, either version 2.1 of the License, or (at your option) any
+# later version.
+#
+# logilab-common is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with logilab-common.  If not, see <http://www.gnu.org/licenses/>.
+"""File and file-path manipulation utilities.
+
+:group path manipulation: first_level_directory, relative_path, is_binary,\
+get_by_ext, remove_dead_links
+:group file manipulation: norm_read, norm_open, lines, stream_lines, lines,\
+write_open_mode, ensure_fs_mode, export
+:sort: path manipulation, file manipulation
+"""
+__docformat__ = "restructuredtext en"
+
+import sys
+import shutil
+import mimetypes
+from os.path import isabs, isdir, islink, split, exists, normpath, join
+from os.path import abspath
+from os import sep, mkdir, remove, listdir, stat, chmod, walk
+from stat import ST_MODE, S_IWRITE
+from cStringIO import StringIO
+
+from logilab.common import STD_BLACKLIST as BASE_BLACKLIST, IGNORED_EXTENSIONS
+from logilab.common.shellutils import find
+from logilab.common.deprecation import deprecated
+from logilab.common.compat import FileIO, any
+
+def first_level_directory(path):
+    """Return the first level directory of a path.
+
+    >>> first_level_directory('home/syt/work')
+    'home'
+    >>> first_level_directory('/home/syt/work')
+    '/'
+    >>> first_level_directory('work')
+    'work'
+    >>>
+
+    :type path: str
+    :param path: the path for which we want the first level directory
+
+    :rtype: str
+    :return: the first level directory appearing in `path`
+    """
+    head, tail = split(path)
+    while head and tail:
+        head, tail = split(head)
+    if tail:
+        return tail
+    # path was absolute, head is the fs root
+    return head
+
+def abspath_listdir(path):
+    """Lists path's content using absolute paths.
+
+    >>> os.listdir('/home')
+    ['adim', 'alf', 'arthur', 'auc']
+    >>> abspath_listdir('/home')
+    ['/home/adim', '/home/alf', '/home/arthur', '/home/auc']
+    """
+    path = abspath(path)
+    return [join(path, filename) for filename in listdir(path)]
+
+
+def is_binary(filename):
+    """Return true if filename may be a binary file, according to it's
+    extension.
+
+    :type filename: str
+    :param filename: the name of the file
+
+    :rtype: bool
+    :return:
+      true if the file is a binary file (actually if it's mime type
+      isn't beginning by text/)
+    """
+    try:
+        return not mimetypes.guess_type(filename)[0].startswith('text')
+    except AttributeError:
+        return 1
+
+
+def write_open_mode(filename):
+    """Return the write mode that should used to open file.
+
+    :type filename: str
+    :param filename: the name of the file
+
+    :rtype: str
+    :return: the mode that should be use to open the file ('w' or 'wb')
+    """
+    if is_binary(filename):
+        return 'wb'
+    return 'w'
+
+
+def ensure_fs_mode(filepath, desired_mode=S_IWRITE):
+    """Check that the given file has the given mode(s) set, else try to
+    set it.
+
+    :type filepath: str
+    :param filepath: path of the file
+
+    :type desired_mode: int
+    :param desired_mode:
+      ORed flags describing the desired mode. Use constants from the
+      `stat` module for file permission's modes
+    """
+    mode = stat(filepath)[ST_MODE]
+    if not mode & desired_mode:
+        chmod(filepath, mode | desired_mode)
+
+
+# XXX (syt) unused? kill?
+class ProtectedFile(FileIO):
+    """A special file-object class that automatically does a 'chmod +w' when
+    needed.
+
+    XXX: for now, the way it is done allows 'normal file-objects' to be
+    created during the ProtectedFile object lifetime.
+    One way to circumvent this would be to chmod / unchmod on each
+    write operation.
+
+    One other way would be to :
+
+    - catch the IOError in the __init__
+
+    - if IOError, then create a StringIO object
+
+    - each write operation writes in this StringIO object
+
+    - on close()/del(), write/append the StringIO content to the file and
+      do the chmod only once
+    """
+    def __init__(self, filepath, mode):
+        self.original_mode = stat(filepath)[ST_MODE]
+        self.mode_changed = False
+        if mode in ('w', 'a', 'wb', 'ab'):
+            if not self.original_mode & S_IWRITE:
+                chmod(filepath, self.original_mode | S_IWRITE)
+                self.mode_changed = True
+        FileIO.__init__(self, filepath, mode)
+
+    def _restore_mode(self):
+        """restores the original mode if needed"""
+        if self.mode_changed:
+            chmod(self.name, self.original_mode)
+            # Don't re-chmod in case of several restore
+            self.mode_changed = False
+
+    def close(self):
+        """restore mode before closing"""
+        self._restore_mode()
+        FileIO.close(self)
+
+    def __del__(self):
+        if not self.closed:
+            self.close()
+
+
+class UnresolvableError(Exception):
+    """Exception raised by relative path when it's unable to compute relative
+    path between two paths.
+    """
+
+def relative_path(from_file, to_file):
+    """Try to get a relative path from `from_file` to `to_file`
+    (path will be absolute if to_file is an absolute file). This function
+    is useful to create link in `from_file` to `to_file`. This typical use
+    case is used in this function description.
+
+    If both files are relative, they're expected to be relative to the same
+    directory.
+
+    >>> relative_path( from_file='toto/index.html', to_file='index.html')
+    '../index.html'
+    >>> relative_path( from_file='index.html', to_file='toto/index.html')
+    'toto/index.html'
+    >>> relative_path( from_file='tutu/index.html', to_file='toto/index.html')
+    '../toto/index.html'
+    >>> relative_path( from_file='toto/index.html', to_file='/index.html')
+    '/index.html'
+    >>> relative_path( from_file='/toto/index.html', to_file='/index.html')
+    '../index.html'
+    >>> relative_path( from_file='/toto/index.html', to_file='/toto/summary.html')
+    'summary.html'
+    >>> relative_path( from_file='index.html', to_file='index.html')
+    ''
+    >>> relative_path( from_file='/index.html', to_file='toto/index.html')
+    Traceback (most recent call last):
+      File "<string>", line 1, in ?
+      File "<stdin>", line 37, in relative_path
+    UnresolvableError
+    >>> relative_path( from_file='/index.html', to_file='/index.html')
+    ''
+    >>>
+
+    :type from_file: str
+    :param from_file: source file (where links will be inserted)
+
+    :type to_file: str
+    :param to_file: target file (on which links point)
+
+    :raise UnresolvableError: if it has been unable to guess a correct path
+
+    :rtype: str
+    :return: the relative path of `to_file` from `from_file`
+    """
+    from_file = normpath(from_file)
+    to_file = normpath(to_file)
+    if from_file == to_file:
+        return ''
+    if isabs(to_file):
+        if not isabs(from_file):
+            return to_file
+    elif isabs(from_file):
+        raise UnresolvableError()
+    from_parts = from_file.split(sep)
+    to_parts = to_file.split(sep)
+    idem = 1
+    result = []
+    while len(from_parts) > 1:
+        dirname = from_parts.pop(0)
+        if idem and len(to_parts) > 1 and dirname == to_parts[0]:
+            to_parts.pop(0)
+        else:
+            idem = 0
+            result.append('..')
+    result += to_parts
+    return sep.join(result)
+
+
+def norm_read(path):
+    """Return the content of the file with normalized line feeds.
+
+    :type path: str
+    :param path: path to the file to read
+
+    :rtype: str
+    :return: the content of the file with normalized line feeds
+    """
+    return open(path, 'U').read()
+norm_read = deprecated("use \"open(path, 'U').read()\"")(norm_read)
+
+def norm_open(path):
+    """Return a stream for a file with content with normalized line feeds.
+
+    :type path: str
+    :param path: path to the file to open
+
+    :rtype: file or StringIO
+    :return: the opened file with normalized line feeds
+    """
+    return open(path, 'U')
+norm_open = deprecated("use \"open(path, 'U')\"")(norm_open)
+
+def lines(path, comments=None):
+    """Return a list of non empty lines in the file located at `path`.
+
+    :type path: str
+    :param path: path to the file
+
+    :type comments: str or None
+    :param comments:
+      optional string which can be used to comment a line in the file
+      (i.e. lines starting with this string won't be returned)
+
+    :rtype: list
+    :return:
+      a list of stripped line in the file, without empty and commented
+      lines
+
+    :warning: at some point this function will probably return an iterator
+    """
+    stream = open(path, 'U')
+    result = stream_lines(stream, comments)
+    stream.close()
+    return result
+
+
+def stream_lines(stream, comments=None):
+    """Return a list of non empty lines in the given `stream`.
+
+    :type stream: object implementing 'xreadlines' or 'readlines'
+    :param stream: file like object
+
+    :type comments: str or None
+    :param comments:
+      optional string which can be used to comment a line in the file
+      (i.e. lines starting with this string won't be returned)
+
+    :rtype: list
+    :return:
+      a list of stripped line in the file, without empty and commented
+      lines
+
+    :warning: at some point this function will probably return an iterator
+    """
+    try:
+        readlines = stream.xreadlines
+    except AttributeError:
+        readlines = stream.readlines
+    result = []
+    for line in readlines():
+        line = line.strip()
+        if line and (comments is None or not line.startswith(comments)):
+            result.append(line)
+    return result
+
+
+def export(from_dir, to_dir,
+           blacklist=BASE_BLACKLIST, ignore_ext=IGNORED_EXTENSIONS,
+           verbose=0):
+    """Make a mirror of `from_dir` in `to_dir`, omitting directories and
+    files listed in the black list or ending with one of the given
+    extensions.
+
+    :type from_dir: str
+    :param from_dir: directory to export
+
+    :type to_dir: str
+    :param to_dir: destination directory
+
+    :type blacklist: list or tuple
+    :param blacklist:
+      list of files or directories to ignore, default to the content of
+      `BASE_BLACKLIST`
+
+    :type ignore_ext: list or tuple
+    :param ignore_ext:
+      list of extensions to ignore, default to  the content of
+      `IGNORED_EXTENSIONS`
+
+    :type verbose: bool
+    :param verbose:
+      flag indicating whether information about exported files should be
+      printed to stderr, default to False
+    """
+    try:
+        mkdir(to_dir)
+    except OSError:
+        pass # FIXME we should use "exists" if the point is about existing dir
+             # else (permission problems?) shouldn't return / raise ?
+    for directory, dirnames, filenames in walk(from_dir):
+        for norecurs in blacklist:
+            try:
+                dirnames.remove(norecurs)
+            except ValueError:
+                continue
+        for dirname in dirnames:
+            src = join(directory, dirname)
+            dest = to_dir + src[len(from_dir):]
+            if isdir(src):
+                if not exists(dest):
+                    mkdir(dest)
+        for filename in filenames:
+            # don't include binary files
+            # endswith does not accept tuple in 2.4
+            if any([filename.endswith(ext) for ext in ignore_ext]):
+                continue
+            src = join(directory, filename)
+            dest = to_dir + src[len(from_dir):]
+            if verbose:
+                print >> sys.stderr, src, '->', dest
+            if exists(dest):
+                remove(dest)
+            shutil.copy2(src, dest)
+
+
+def remove_dead_links(directory, verbose=0):
+    """Recursively traverse directory and remove all dead links.
+
+    :type directory: str
+    :param directory: directory to cleanup
+
+    :type verbose: bool
+    :param verbose:
+      flag indicating whether information about deleted links should be
+      printed to stderr, default to False
+    """
+    for dirpath, dirname, filenames in walk(directory):
+        for filename in dirnames + filenames:
+            src = join(dirpath, filename)
+            if islink(src) and not exists(src):
+                if verbose:
+                    print 'remove dead link', src
+                remove(src)
+

pylibs/logilab/common/graph.py

@@ -0,0 +1,273 @@
+# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+#
+# This file is part of logilab-common.
+#
+# logilab-common is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation, either version 2.1 of the License, or (at your option) any
+# later version.
+#
+# logilab-common is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with logilab-common.  If not, see <http://www.gnu.org/licenses/>.
+"""Graph manipulation utilities.
+
+(dot generation adapted from pypy/translator/tool/make_dot.py)
+"""
+
+__docformat__ = "restructuredtext en"
+
+__metaclass__ = type
+
+import os.path as osp
+import os
+import sys
+import tempfile
+from logilab.common.compat import str_encode
+
+def escape(value):
+    """Make <value> usable in a dot file."""
+    lines = [line.replace('"', '\\"') for line in value.split('\n')]
+    data = '\\l'.join(lines)
+    return '\\n' + data
+
+def target_info_from_filename(filename):
+    """Transforms /some/path/foo.png into ('/some/path', 'foo.png', 'png')."""
+    basename = osp.basename(filename)
+    storedir = osp.dirname(osp.abspath(filename))
+    target = filename.split('.')[-1]
+    return storedir, basename, target
+
+
+class DotBackend:
+    """Dot File backend."""
+    def __init__(self, graphname, rankdir=None, size=None, ratio=None,
+            charset='utf-8', renderer='dot', additionnal_param={}):
+        self.graphname = graphname
+        self.renderer = renderer
+        self.lines = []
+        self._source = None
+        self.emit("digraph %s {" % normalize_node_id(graphname))
+        if rankdir:
+            self.emit('rankdir=%s' % rankdir)
+        if ratio:
+            self.emit('ratio=%s' % ratio)
+        if size:
+            self.emit('size="%s"' % size)
+        if charset:
+            assert charset.lower() in ('utf-8', 'iso-8859-1', 'latin1'), \
+                   'unsupported charset %s' % charset
+            self.emit('charset="%s"' % charset)
+        for param in additionnal_param.iteritems():
+            self.emit('='.join(param))
+
+    def get_source(self):
+        """returns self._source"""
+        if self._source is None:
+            self.emit("}\n")
+            self._source = '\n'.join(self.lines)
+            del self.lines
+        return self._source
+
+    source = property(get_source)
+
+    def generate(self, outputfile=None, dotfile=None, mapfile=None):
+        """Generates a graph file.
+
+        :param outputfile: filename and path [defaults to graphname.png]
+        :param dotfile: filename and path [defaults to graphname.dot]
+
+        :rtype: str
+        :return: a path to the generated file
+        """
+        import subprocess # introduced in py 2.4
+        name = self.graphname
+        if not dotfile:
+            # if 'outputfile' is a dot file use it as 'dotfile'
+            if outputfile and outputfile.endswith(".dot"):
+                dotfile = outputfile
+            else:
+                dotfile = '%s.dot' % name
+        if outputfile is not None:
+            storedir, basename, target = target_info_from_filename(outputfile)
+            if target != "dot":
+                pdot, dot_sourcepath = tempfile.mkstemp(".dot", name)
+                os.close(pdot)
+            else:
+                dot_sourcepath = osp.join(storedir, dotfile)
+        else:
+            target = 'png'
+            pdot, dot_sourcepath = tempfile.mkstemp(".dot", name)
+            ppng, outputfile = tempfile.mkstemp(".png", name)
+            os.close(pdot)
+            os.close(ppng)
+        pdot = open(dot_sourcepath, 'w')
+        pdot.write(str_encode(self.source, 'utf8'))
+        pdot.close()
+        if target != 'dot':
+            if sys.platform == 'win32':
+                use_shell = True
+            else:
+                use_shell = False
+            if mapfile:
+                subprocess.call([self.renderer,  '-Tcmapx', '-o', mapfile, '-T', target, dot_sourcepath, '-o', outputfile],
+                                shell=use_shell)
+            else:
+                subprocess.call([self.renderer, '-T',  target,
+                                 dot_sourcepath, '-o',  outputfile],
+                                shell=use_shell)
+            os.unlink(dot_sourcepath)
+        return outputfile
+
+    def emit(self, line):
+        """Adds <line> to final output."""
+        self.lines.append(line)
+
+    def emit_edge(self, name1, name2, **props):
+        """emit an edge from <name1> to <name2>.
+        edge properties: see http://www.graphviz.org/doc/info/attrs.html
+        """
+        attrs = ['%s="%s"' % (prop, value) for prop, value in props.items()]
+        n_from, n_to = normalize_node_id(name1), normalize_node_id(name2)
+        self.emit('%s -> %s [%s];' % (n_from, n_to, ", ".join(attrs)) )
+
+    def emit_node(self, name, **props):
+        """emit a node with given properties.
+        node properties: see http://www.graphviz.org/doc/info/attrs.html
+        """
+        attrs = ['%s="%s"' % (prop, value) for prop, value in props.items()]
+        self.emit('%s [%s];' % (normalize_node_id(name), ", ".join(attrs)))
+
+def normalize_node_id(nid):
+    """Returns a suitable DOT node id for `nid`."""
+    return '"%s"' % nid
+
+class GraphGenerator:
+    def __init__(self, backend):
+        # the backend is responsible to output the graph in a particular format
+        self.backend = backend
+
+    # XXX doesn't like space in outpufile / mapfile
+    def generate(self, visitor, propshdlr, outputfile=None, mapfile=None):
+        # the visitor
+        # the property handler is used to get node and edge properties
+        # according to the graph and to the backend
+        self.propshdlr = propshdlr
+        for nodeid, node in visitor.nodes():
+            props = propshdlr.node_properties(node)
+            self.backend.emit_node(nodeid, **props)
+        for subjnode, objnode, edge in visitor.edges():
+            props = propshdlr.edge_properties(edge, subjnode, objnode)
+            self.backend.emit_edge(subjnode, objnode, **props)
+        return self.backend.generate(outputfile=outputfile, mapfile=mapfile)
+
+
+class UnorderableGraph(Exception):
+    pass
+
+def ordered_nodes(graph):
+    """takes a dependency graph dict as arguments and return an ordered tuple of
+    nodes starting with nodes without dependencies and up to the outermost node.
+
+    If there is some cycle in the graph, :exc:`UnorderableGraph` will be raised.
+
+    Also the given graph dict will be emptied.
+    """
+    # check graph consistency
+    cycles = get_cycles(graph)
+    if cycles:
+        cycles = '\n'.join([' -> '.join(cycle) for cycle in cycles])
+        raise UnorderableGraph('cycles in graph: %s' % cycles)
+    vertices = set(graph)
+    to_vertices = set()
+    for edges in graph.values():
+        to_vertices |= set(edges)
+    missing_vertices = to_vertices - vertices
+    if missing_vertices:
+        raise UnorderableGraph('missing vertices: %s' % ', '.join(missing_vertices))
+    # order vertices
+    order = []
+    order_set = set()
+    old_len = None
+    while graph:
+        if old_len == len(graph):
+            raise UnorderableGraph('unknown problem with %s' % graph)
+        old_len = len(graph)
+        deps_ok = []
+        for node, node_deps in graph.items():
+            for dep in node_deps:
+                if dep not in order_set:
+                    break
+            else:
+                deps_ok.append(node)
+        order.append(deps_ok)
+        order_set |= set(deps_ok)
+        for node in deps_ok:
+            del graph[node]
+    result = []
+    for grp in reversed(order):
+        result.extend(sorted(grp))
+    return tuple(result)
+
+
+def get_cycles(graph_dict, vertices=None):
+    '''given a dictionary representing an ordered graph (i.e. key are vertices
+    and values is a list of destination vertices representing edges), return a
+    list of detected cycles
+    '''
+    if not graph_dict:
+        return ()
+    result = []
+    if vertices is None:
+        vertices = graph_dict.keys()
+    for vertice in vertices:
+        _get_cycles(graph_dict, vertice, [], result)
+    return result
+
+def _get_cycles(graph_dict, vertice=None, path=None, result=None):
+    """recursive function doing the real work for get_cycles"""
+    if vertice in path:
+        cycle = [vertice]
+        for node in path[::-1]:
+            if node == vertice:
+                break
+            cycle.insert(0, node)
+        # make a canonical representation
+        start_from = min(cycle)
+        index = cycle.index(start_from)
+        cycle = cycle[index:] + cycle[0:index]
+        # append it to result if not already in
+        if not cycle in result:
+            result.append(cycle)
+        return
+    path.append(vertice)
+    try:
+        for node in graph_dict[vertice]:
+            _get_cycles(graph_dict, node, path, result)
+    except KeyError:
+        pass
+    path.pop()
+
+def has_path(graph_dict, fromnode, tonode, path=None):
+    """generic function taking a simple graph definition as a dictionary, with
+    node has key associated to a list of nodes directly reachable from it.
+
+    Return None if no path exists to go from `fromnode` to `tonode`, else the
+    first path found (as a list including the destination node at last)
+    """
+    if path is None:
+        path = []
+    elif fromnode in path:
+        return None
+    path.append(fromnode)
+    for destnode in graph_dict[fromnode]:
+        if destnode == tonode or has_path(graph_dict, destnode, tonode, path):
+            return path[1:] + [tonode]
+    path.pop()
+    return None
+

pylibs/logilab/common/hg.py

@@ -0,0 +1,130 @@
+# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+#
+# This file is part of logilab-common.
+#
+# logilab-common is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation, either version 2.1 of the License, or (at your option) any
+# later version.
+#
+# logilab-common is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with logilab-common.  If not, see <http://www.gnu.org/licenses/>.
+"""mercurial utilities (mercurial should be installed)"""
+
+__docformat__ = "restructuredtext en"
+
+import os
+import sys
+import os.path as osp
+
+try:
+    from mercurial.error import RepoError
+    from mercurial.__version__ import version as hg_version
+except ImportError:
+    from mercurial.repo import RepoError
+    from mercurial.version import get_version
+    hg_version = get_version()
+
+from mercurial.hg import repository as Repository
+from mercurial.ui import ui as Ui
+from mercurial.node import short
+try:
+    # mercurial >= 1.2 (?)
+    from mercurial.cmdutil import walkchangerevs
+except ImportError, ex:
+    from mercurial.commands import walkchangerevs
+try:
+    # mercurial >= 1.1 (.1?)
+    from mercurial.util import cachefunc
+except ImportError, ex:
+    def cachefunc(func):
+        return func
+try:
+    # mercurial >= 1.3.1
+    from mercurial import encoding
+    _encoding = encoding.encoding
+except ImportError:
+    try:
+        from mercurial.util import _encoding
+    except ImportError:
+        import locale
+        # stay compatible with mercurial 0.9.1 (etch debian release)
+        # (borrowed from mercurial.util 1.1.2)
+        try:
+            _encoding = os.environ.get("HGENCODING")
+            if sys.platform == 'darwin' and not _encoding:
+                # On darwin, getpreferredencoding ignores the locale environment and
+                # always returns mac-roman. We override this if the environment is
+                # not C (has been customized by the user).
+                locale.setlocale(locale.LC_CTYPE, '')
+                _encoding = locale.getlocale()[1]
+            if not _encoding:
+                _encoding = locale.getpreferredencoding() or 'ascii'
+        except locale.Error:
+            _encoding = 'ascii'
+try:
+    # demandimport causes problems when activated, ensure it isn't
+    # XXX put this in apycot where the pb has been noticed?
+    from mercurial import demandimport
+    demandimport.disable()
+except:
+    pass
+
+Ui.warn = lambda *args, **kwargs: 0 # make it quiet
+
+def find_repository(path):
+    """returns <path>'s mercurial repository
+
+    None if <path> is not under hg control
+    """
+    path = osp.realpath(osp.abspath(path))
+    while not osp.isdir(osp.join(path, ".hg")):
+        oldpath = path
+        path = osp.dirname(path)
+        if path == oldpath:
+            return None
+    return path
+
+
+def get_repository(path):
+    """Simple function that open a hg repository"""
+    repopath = find_repository(path)
+    if repopath is None:
+        raise RuntimeError('no repository found in %s' % osp.abspath(path))
+    return Repository(Ui(), path=repopath)
+
+def incoming(wdrepo, masterrepo):
+    try:
+        return wdrepo.findincoming(masterrepo)
+    except AttributeError:
+        from mercurial import hg, discovery
+        revs, checkout = hg.addbranchrevs(wdrepo, masterrepo, ('', []), None)
+        common, incoming, rheads = discovery.findcommonincoming(
+            wdrepo, masterrepo, heads=revs)
+        if not masterrepo.local():
+            from mercurial import bundlerepo, changegroup
+            if revs is None and masterrepo.capable('changegroupsubset'):
+                revs = rheads
+            if revs is None:
+                cg = masterrepo.changegroup(incoming, "incoming")
+            else:
+                cg = masterrepo.changegroupsubset(incoming, revs, 'incoming')
+            fname = changegroup.writebundle(cg, None, "HG10UN")
+            # use the created uncompressed bundlerepo
+            masterrepo = bundlerepo.bundlerepository(wdrepo.ui, wdrepo.root, fname)
+        return masterrepo.changelog.nodesbetween(incoming, revs)[0]
+
+def outgoing(wdrepo, masterrepo):
+    try:
+        return wdrepo.findoutgoing(masterrepo)
+    except AttributeError:
+        from mercurial import hg, discovery
+        revs, checkout = hg.addbranchrevs(wdrepo, wdrepo, ('', []), None)
+        o = discovery.findoutgoing(wdrepo, masterrepo)
+        return wdrepo.changelog.nodesbetween(o, revs)[0]

pylibs/logilab/common/interface.py

@@ -0,0 +1,71 @@
+# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+#
+# This file is part of logilab-common.
+#
+# logilab-common is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation, either version 2.1 of the License, or (at your option) any
+# later version.
+#
+# logilab-common is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with logilab-common.  If not, see <http://www.gnu.org/licenses/>.
+"""Bases class for interfaces to provide 'light' interface handling.
+
+ TODO:
+  _ implements a check method which check that an object implements the
+    interface
+  _ Attribute objects
+
+  This module requires at least python 2.2
+"""
+__docformat__ = "restructuredtext en"
+
+
+class Interface(object):
+    """Base class for interfaces."""
+    def is_implemented_by(cls, instance):
+        return implements(instance, cls)
+    is_implemented_by = classmethod(is_implemented_by)
+
+
+def implements(obj, interface):
+    """Return true if the give object (maybe an instance or class) implements
+    the interface.
+    """
+    kimplements = getattr(obj, '__implements__', ())
+    if not isinstance(kimplements, (list, tuple)):
+        kimplements = (kimplements,)
+    for implementedinterface in kimplements:
+        if issubclass(implementedinterface, interface):
+            return True
+    return False
+
+
+def extend(klass, interface, _recurs=False):
+    """Add interface to klass'__implements__ if not already implemented in.
+
+    If klass is subclassed, ensure subclasses __implements__ it as well.
+
+    NOTE: klass should be e new class.
+    """
+    if not implements(klass, interface):
+        try:
+            kimplements = klass.__implements__
+            kimplementsklass = type(kimplements)
+            kimplements = list(kimplements)
+        except AttributeError:
+            kimplementsklass = tuple
+            kimplements = []
+        kimplements.append(interface)
+        klass.__implements__ = kimplementsklass(kimplements)
+        for subklass in klass.__subclasses__():
+            extend(subklass, interface, _recurs=True)
+    elif _recurs:
+        for subklass in klass.__subclasses__():
+            extend(subklass, interface, _recurs=True)

pylibs/logilab/common/logging_ext.py

@@ -0,0 +1,178 @@
+# -*- coding: utf-8 -*-
+# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+#
+# This file is part of logilab-common.
+#
+# logilab-common is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation, either version 2.1 of the License, or (at your option) any
+# later version.
+#
+# logilab-common is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with logilab-common.  If not, see <http://www.gnu.org/licenses/>.
+"""Extends the logging module from the standard library."""
+
+__docformat__ = "restructuredtext en"
+
+import os
+import sys
+import logging
+
+from logilab.common.textutils import colorize_ansi
+
+
+def set_log_methods(cls, logger):
+    """bind standard logger's methods as methods on the class"""
+    cls.__logger = logger
+    for attr in ('debug', 'info', 'warning', 'error', 'critical', 'exception'):
+        setattr(cls, attr, getattr(logger, attr))
+
+
+def xxx_cyan(record):
+    if 'XXX' in record.message:
+        return 'cyan'
+
+class ColorFormatter(logging.Formatter):
+    """
+    A color Formatter for the logging standard module.
+
+    By default, colorize CRITICAL and ERROR in red, WARNING in orange, INFO in
+    green and DEBUG in yellow.
+
+    self.colors is customizable via the 'color' constructor argument (dictionary).
+
+    self.colorfilters is a list of functions that get the LogRecord
+    and return a color name or None.
+    """
+
+    def __init__(self, fmt=None, datefmt=None, colors=None):
+        logging.Formatter.__init__(self, fmt, datefmt)
+        self.colorfilters = []
+        self.colors = {'CRITICAL': 'red',
+                       'ERROR': 'red',
+                       'WARNING': 'magenta',
+                       'INFO': 'green',
+                       'DEBUG': 'yellow',
+                       }
+        if colors is not None:
+            assert isinstance(colors, dict)
+            self.colors.update(colors)
+
+    def format(self, record):
+        msg = logging.Formatter.format(self, record)
+        if record.levelname in self.colors:
+            color = self.colors[record.levelname]
+            return colorize_ansi(msg, color)
+        else:
+            for cf in self.colorfilters:
+                color = cf(record)
+                if color:
+                    return colorize_ansi(msg, color)
+        return msg
+
+def set_color_formatter(logger=None, **kw):
+    """
+    Install a color formatter on the 'logger'. If not given, it will
+    defaults to the default logger.
+
+    Any additional keyword will be passed as-is to the ColorFormatter
+    constructor.
+    """
+    if logger is None:
+        logger = logging.getLogger()
+        if not logger.handlers:
+            logging.basicConfig()
+    format_msg = logger.handlers[0].formatter._fmt
+    fmt = ColorFormatter(format_msg, **kw)
+    fmt.colorfilters.append(xxx_cyan)
+    logger.handlers[0].setFormatter(fmt)
+
+
+LOG_FORMAT = '%(asctime)s - (%(name)s) %(levelname)s: %(message)s'
+LOG_DATE_FORMAT = '%Y-%m-%d %H:%M:%S'
+
+def get_handler(debug=False, syslog=False, logfile=None, rotation_parameters=None):
+    """get an apropriate handler according to given parameters"""
+    if os.environ.get('APYCOT_ROOT'):
+        handler = logging.StreamHandler(sys.stdout)
+    if debug:
+        handler = logging.StreamHandler()
+    elif logfile is None:
+        if syslog:
+            from logging import handlers
+            handler = handlers.SysLogHandler()
+        else:
+            handler = logging.StreamHandler()
+    else:
+        try:
+            if rotation_parameters is None:
+                handler = logging.FileHandler(logfile)
+            else:
+                from logging.handlers import TimedRotatingFileHandler
+                handler = TimedRotatingFileHandler(
+                    logfile, **rotation_parameters)
+        except IOError:
+            handler = logging.StreamHandler()
+    return handler
+
+def get_threshold(debug=False, logthreshold=None):
+    if logthreshold is None:
+        if debug:
+            logthreshold = logging.DEBUG
+        else:
+            logthreshold = logging.ERROR
+    elif isinstance(logthreshold, basestring):
+        logthreshold = getattr(logging, THRESHOLD_MAP.get(logthreshold,
+                                                          logthreshold))
+    return logthreshold
+
+def get_formatter(logformat=LOG_FORMAT, logdateformat=LOG_DATE_FORMAT):
+    isatty = hasattr(sys.__stdout__, 'isatty') and sys.__stdout__.isatty()
+    if isatty and sys.platform != 'win32':
+        fmt = ColorFormatter(logformat, logdateformat)
+        def col_fact(record):
+            if 'XXX' in record.message:
+                return 'cyan'
+            if 'kick' in record.message:
+                return 'red'
+        fmt.colorfilters.append(col_fact)
+    else:
+        fmt = logging.Formatter(logformat, logdateformat)
+    return fmt
+
+def init_log(debug=False, syslog=False, logthreshold=None, logfile=None,
+             logformat=LOG_FORMAT, logdateformat=LOG_DATE_FORMAT, fmt=None,
+             rotation_parameters=None, handler=None):
+    """init the log service"""
+    logger = logging.getLogger()
+    if handler is None:
+        handler = get_handler(debug, syslog, logfile, rotation_parameters)
+    # only addHandler and removeHandler method while I would like a setHandler
+    # method, so do it this way :$
+    logger.handlers = [handler]
+    logthreshold = get_threshold(debug, logthreshold)
+    logger.setLevel(logthreshold)
+    if fmt is None:
+        if debug:
+            fmt = get_formatter(logformat=logformat, logdateformat=logdateformat)
+        else:
+            fmt = logging.Formatter(logformat, logdateformat)
+    handler.setFormatter(fmt)
+    return handler
+
+# map logilab.common.logger thresholds to logging thresholds
+THRESHOLD_MAP = {'LOG_DEBUG':   'DEBUG',
+                 'LOG_INFO':    'INFO',
+                 'LOG_NOTICE':  'INFO',
+                 'LOG_WARN':    'WARNING',
+                 'LOG_WARNING': 'WARNING',
+                 'LOG_ERR':     'ERROR',
+                 'LOG_ERROR':   'ERROR',
+                 'LOG_CRIT':    'CRITICAL',
+                 }

pylibs/logilab/common/modutils.py

@@ -0,0 +1,653 @@
+# -*- coding: utf-8 -*-
+# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+#
+# This file is part of logilab-common.
+#
+# logilab-common is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation, either version 2.1 of the License, or (at your option) any
+# later version.
+#
+# logilab-common is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with logilab-common.  If not, see <http://www.gnu.org/licenses/>.
+"""Python modules manipulation utility functions.
+
+:type PY_SOURCE_EXTS: tuple(str)
+:var PY_SOURCE_EXTS: list of possible python source file extension
+
+:type STD_LIB_DIR: str
+:var STD_LIB_DIR: directory where standard modules are located
+
+:type BUILTIN_MODULES: dict
+:var BUILTIN_MODULES: dictionary with builtin module names has key
+"""
+__docformat__ = "restructuredtext en"
+
+import sys
+import os
+from os.path import splitext, join, abspath, isdir, dirname, exists, basename
+from imp import find_module, load_module, C_BUILTIN, PY_COMPILED, PKG_DIRECTORY
+from distutils.sysconfig import get_config_var, get_python_lib, get_python_version
+
+try:
+    import zipimport
+except ImportError:
+    zipimport = None
+
+ZIPFILE = object()
+
+from logilab.common import STD_BLACKLIST, _handle_blacklist
+
+# Notes about STD_LIB_DIR
+# Consider arch-specific installation for STD_LIB_DIR definition
+# :mod:`distutils.sysconfig` contains to much hardcoded values to rely on
+#
+# :see: `Problems with /usr/lib64 builds <http://bugs.python.org/issue1294959>`_
+# :see: `FHS <http://www.pathname.com/fhs/pub/fhs-2.3.html#LIBLTQUALGTALTERNATEFORMATESSENTIAL>`_
+if sys.platform.startswith('win'):
+    PY_SOURCE_EXTS = ('py', 'pyw')
+    PY_COMPILED_EXTS = ('dll', 'pyd')
+    STD_LIB_DIR = get_python_lib(standard_lib=1)
+else:
+    PY_SOURCE_EXTS = ('py',)
+    PY_COMPILED_EXTS = ('so',)
+    # extend lib dir with some arch-dependant paths
+    STD_LIB_DIR = join(get_config_var("LIBDIR"), "python%s" % get_python_version())
+
+BUILTIN_MODULES = dict(zip(sys.builtin_module_names,
+                           [1]*len(sys.builtin_module_names)))
+
+
+class NoSourceFile(Exception):
+    """exception raised when we are not able to get a python
+    source file for a precompiled file
+    """
+
+class LazyObject(object):
+    def __init__(self, module, obj):
+        self.module = module
+        self.obj = obj
+        self._imported = None
+
+    def _getobj(self):
+        if self._imported is None:
+           self._imported = getattr(load_module_from_name(self.module),
+                                    self.obj)
+        return self._imported
+
+    def __getattribute__(self, attr):
+        try:
+            return super(LazyObject, self).__getattribute__(attr)
+        except AttributeError, ex:
+            return getattr(self._getobj(), attr)
+
+    def __call__(self, *args, **kwargs):
+        return self._getobj()(*args, **kwargs)
+
+
+def load_module_from_name(dotted_name, path=None, use_sys=1):
+    """Load a Python module from its name.
+
+    :type dotted_name: str
+    :param dotted_name: python name of a module or package
+
+    :type path: list or None
+    :param path:
+      optional list of path where the module or package should be
+      searched (use sys.path if nothing or None is given)
+
+    :type use_sys: bool
+    :param use_sys:
+      boolean indicating whether the sys.modules dictionary should be
+      used or not
+
+
+    :raise ImportError: if the module or package is not found
+
+    :rtype: module
+    :return: the loaded module
+    """
+    return load_module_from_modpath(dotted_name.split('.'), path, use_sys)
+
+
+def load_module_from_modpath(parts, path=None, use_sys=1):
+    """Load a python module from its splitted name.
+
+    :type parts: list(str) or tuple(str)
+    :param parts:
+      python name of a module or package splitted on '.'
+
+    :type path: list or None
+    :param path:
+      optional list of path where the module or package should be
+      searched (use sys.path if nothing or None is given)
+
+    :type use_sys: bool
+    :param use_sys:
+      boolean indicating whether the sys.modules dictionary should be used or not
+
+    :raise ImportError: if the module or package is not found
+
+    :rtype: module
+    :return: the loaded module
+    """
+    if use_sys:
+        try:
+            return sys.modules['.'.join(parts)]
+        except KeyError:
+            pass
+    modpath = []
+    prevmodule = None
+    for part in parts:
+        modpath.append(part)
+        curname = '.'.join(modpath)
+        module = None
+        if len(modpath) != len(parts):
+            # even with use_sys=False, should try to get outer packages from sys.modules
+            module = sys.modules.get(curname)
+        if module is None:
+            mp_file, mp_filename, mp_desc = find_module(part, path)
+            module = load_module(curname, mp_file, mp_filename, mp_desc)
+        if prevmodule:
+            setattr(prevmodule, part, module)
+        _file = getattr(module, '__file__', '')
+        if not _file and len(modpath) != len(parts):
+            raise ImportError('no module in %s' % '.'.join(parts[len(modpath):]) )
+        path = [dirname( _file )]
+        prevmodule = module
+    return module
+
+
+def load_module_from_file(filepath, path=None, use_sys=1, extrapath=None):
+    """Load a Python module from it's path.
+
+    :type filepath: str
+    :param filepath: path to the python module or package
+
+    :type path: list or None
+    :param path:
+      optional list of path where the module or package should be
+      searched (use sys.path if nothing or None is given)
+
+    :type use_sys: bool
+    :param use_sys:
+      boolean indicating whether the sys.modules dictionary should be
+      used or not
+
+
+    :raise ImportError: if the module or package is not found
+
+    :rtype: module
+    :return: the loaded module
+    """
+    modpath = modpath_from_file(filepath, extrapath)
+    return load_module_from_modpath(modpath, path, use_sys)
+
+
+def _check_init(path, mod_path):
+    """check there are some __init__.py all along the way"""
+    for part in mod_path:
+        path = join(path, part)
+        if not _has_init(path):
+            return False
+    return True
+
+
+def modpath_from_file(filename, extrapath=None):
+    """given a file path return the corresponding splitted module's name
+    (i.e name of a module or package splitted on '.')
+
+    :type filename: str
+    :param filename: file's path for which we want the module's name
+
+    :type extrapath: dict
+    :param extrapath:
+      optional extra search path, with path as key and package name for the path
+      as value. This is usually useful to handle package splitted in multiple
+      directories using __path__ trick.
+
+
+    :raise ImportError:
+      if the corresponding module's name has not been found
+
+    :rtype: list(str)
+    :return: the corresponding splitted module's name
+    """
+    base = splitext(abspath(filename))[0]
+    if extrapath is not None:
+        for path_ in extrapath:
+            path = abspath(path_)
+            if path and base[:len(path)] == path:
+                submodpath = [pkg for pkg in base[len(path):].split(os.sep)
+                              if pkg]
+                if _check_init(path, submodpath[:-1]):
+                    return extrapath[path_].split('.') + submodpath
+    for path in sys.path:
+        path = abspath(path)
+        if path and base[:len(path)] == path:
+            if filename.find('site-packages') != -1 and \
+                   path.find('site-packages') == -1:
+                continue
+            modpath = [pkg for pkg in base[len(path):].split(os.sep) if pkg]
+            if _check_init(path, modpath[:-1]):
+                return modpath
+    raise ImportError('Unable to find module for %s in %s' % (
+        filename, ', \n'.join(sys.path)))
+
+
+
+def file_from_modpath(modpath, path=None, context_file=None):
+    """given a mod path (i.e. splitted module / package name), return the
+    corresponding file, giving priority to source file over precompiled
+    file if it exists
+
+    :type modpath: list or tuple
+    :param modpath:
+      splitted module's name (i.e name of a module or package splitted
+      on '.')
+      (this means explicit relative imports that start with dots have
+      empty strings in this list!)
+
+    :type path: list or None
+    :param path:
+      optional list of path where the module or package should be
+      searched (use sys.path if nothing or None is given)
+
+    :type context_file: str or None
+    :param context_file:
+      context file to consider, necessary if the identifier has been
+      introduced using a relative import unresolvable in the actual
+      context (i.e. modutils)
+
+    :raise ImportError: if there is no such module in the directory
+
+    :rtype: str or None
+    :return:
+      the path to the module's file or None if it's an integrated
+      builtin module such as 'sys'
+    """
+    if context_file is not None:
+        context = dirname(context_file)
+    else:
+        context = context_file
+    if modpath[0] == 'xml':
+        # handle _xmlplus
+        try:
+            return _file_from_modpath(['_xmlplus'] + modpath[1:], path, context)
+        except ImportError:
+            return _file_from_modpath(modpath, path, context)
+    elif modpath == ['os', 'path']:
+        # FIXME: currently ignoring search_path...
+        return os.path.__file__
+    return _file_from_modpath(modpath, path, context)
+
+
+
+def get_module_part(dotted_name, context_file=None):
+    """given a dotted name return the module part of the name :
+
+    >>> get_module_part('logilab.common.modutils.get_module_part')
+    'logilab.common.modutils'
+
+    :type dotted_name: str
+    :param dotted_name: full name of the identifier we are interested in
+
+    :type context_file: str or None
+    :param context_file:
+      context file to consider, necessary if the identifier has been
+      introduced using a relative import unresolvable in the actual
+      context (i.e. modutils)
+
+
+    :raise ImportError: if there is no such module in the directory
+
+    :rtype: str or None
+    :return:
+      the module part of the name or None if we have not been able at
+      all to import the given name
+
+    XXX: deprecated, since it doesn't handle package precedence over module
+    (see #10066)
+    """
+    # os.path trick
+    if dotted_name.startswith('os.path'):
+        return 'os.path'
+    parts = dotted_name.split('.')
+    if context_file is not None:
+        # first check for builtin module which won't be considered latter
+        # in that case (path != None)
+        if parts[0] in BUILTIN_MODULES:
+            if len(parts) > 2:
+                raise ImportError(dotted_name)
+            return parts[0]
+        # don't use += or insert, we want a new list to be created !
+    path = None
+    starti = 0
+    if parts[0] == '':
+        assert context_file is not None, \
+                'explicit relative import, but no context_file?'
+        path = [] # prevent resolving the import non-relatively
+        starti = 1
+    while parts[starti] == '': # for all further dots: change context
+        starti += 1
+        context_file = dirname(context_file)
+    for i in range(starti, len(parts)):
+        try:
+            file_from_modpath(parts[starti:i+1],
+                    path=path, context_file=context_file)
+        except ImportError:
+            if not i >= max(1, len(parts) - 2):
+                raise
+            return '.'.join(parts[:i])
+    return dotted_name
+
+
+def get_modules(package, src_directory, blacklist=STD_BLACKLIST):
+    """given a package directory return a list of all available python
+    modules in the package and its subpackages
+
+    :type package: str
+    :param package: the python name for the package
+
+    :type src_directory: str
+    :param src_directory:
+      path of the directory corresponding to the package
+
+    :type blacklist: list or tuple
+    :param blacklist:
+      optional list of files or directory to ignore, default to
+      the value of `logilab.common.STD_BLACKLIST`
+
+    :rtype: list
+    :return:
+      the list of all available python modules in the package and its
+      subpackages
+    """
+    modules = []
+    for directory, dirnames, filenames in os.walk(src_directory):
+        _handle_blacklist(blacklist, dirnames, filenames)
+        # check for __init__.py
+        if not '__init__.py' in filenames:
+            dirnames[:] = ()
+            continue
+        if directory != src_directory:
+            dir_package = directory[len(src_directory):].replace(os.sep, '.')
+            modules.append(package + dir_package)
+        for filename in filenames:
+            if _is_python_file(filename) and filename != '__init__.py':
+                src = join(directory, filename)
+                module = package + src[len(src_directory):-3]
+                modules.append(module.replace(os.sep, '.'))
+    return modules
+
+
+
+def get_module_files(src_directory, blacklist=STD_BLACKLIST):
+    """given a package directory return a list of all available python
+    module's files in the package and its subpackages
+
+    :type src_directory: str
+    :param src_directory:
+      path of the directory corresponding to the package
+
+    :type blacklist: list or tuple
+    :param blacklist:
+      optional list of files or directory to ignore, default to the value of
+      `logilab.common.STD_BLACKLIST`
+
+    :rtype: list
+    :return:
+      the list of all available python module's files in the package and
+      its subpackages
+    """
+    files = []
+    for directory, dirnames, filenames in os.walk(src_directory):
+        _handle_blacklist(blacklist, dirnames, filenames)
+        # check for __init__.py
+        if not '__init__.py' in filenames:
+            dirnames[:] = ()
+            continue
+        for filename in filenames:
+            if _is_python_file(filename):
+                src = join(directory, filename)
+                files.append(src)
+    return files
+
+
+def get_source_file(filename, include_no_ext=False):
+    """given a python module's file name return the matching source file
+    name (the filename will be returned identically if it's a already an
+    absolute path to a python source file...)
+
+    :type filename: str
+    :param filename: python module's file name
+
+
+    :raise NoSourceFile: if no source file exists on the file system
+
+    :rtype: str
+    :return: the absolute path of the source file if it exists
+    """
+    base, orig_ext = splitext(abspath(filename))
+    for ext in PY_SOURCE_EXTS:
+        source_path = '%s.%s' % (base, ext)
+        if exists(source_path):
+            return source_path
+    if include_no_ext and not orig_ext and exists(base):
+        return base
+    raise NoSourceFile(filename)
+
+
+def cleanup_sys_modules(directories):
+    """remove submodules of `directories` from `sys.modules`"""
+    for modname, module in sys.modules.items():
+        modfile = getattr(module, '__file__', None)
+        if modfile:
+            for directory in directories:
+                if modfile.startswith(directory):
+                    del sys.modules[modname]
+                    break
+
+
+def is_python_source(filename):
+    """
+    rtype: bool
+    return: True if the filename is a python source file
+    """
+    return splitext(filename)[1][1:] in PY_SOURCE_EXTS
+
+
+
+def is_standard_module(modname, std_path=(STD_LIB_DIR,)):
+    """try to guess if a module is a standard python module (by default,
+    see `std_path` parameter's description)
+
+    :type modname: str
+    :param modname: name of the module we are interested in
+
+    :type std_path: list(str) or tuple(str)
+    :param std_path: list of path considered has standard
+
+
+    :rtype: bool
+    :return:
+      true if the module:
+      - is located on the path listed in one of the directory in `std_path`
+      - is a built-in module
+    """
+    modname = modname.split('.')[0]
+    try:
+        filename = file_from_modpath([modname])
+    except ImportError, ex:
+        # import failed, i'm probably not so wrong by supposing it's
+        # not standard...
+        return 0
+    # modules which are not living in a file are considered standard
+    # (sys and __builtin__ for instance)
+    if filename is None:
+        return 1
+    filename = abspath(filename)
+    for path in std_path:
+        path = abspath(path)
+        if filename.startswith(path):
+            pfx_len = len(path)
+            if filename[pfx_len+1:pfx_len+14] != 'site-packages':
+                return 1
+            return 0
+    return False
+
+
+
+def is_relative(modname, from_file):
+    """return true if the given module name is relative to the given
+    file name
+
+    :type modname: str
+    :param modname: name of the module we are interested in
+
+    :type from_file: str
+    :param from_file:
+      path of the module from which modname has been imported
+
+    :rtype: bool
+    :return:
+      true if the module has been imported relatively to `from_file`
+    """
+    if not isdir(from_file):
+        from_file = dirname(from_file)
+    if from_file in sys.path:
+        return False
+    try:
+        find_module(modname.split('.')[0], [from_file])
+        return True
+    except ImportError:
+        return False
+
+
+# internal only functions #####################################################
+
+def _file_from_modpath(modpath, path=None, context=None):
+    """given a mod path (i.e. splitted module / package name), return the
+    corresponding file
+
+    this function is used internally, see `file_from_modpath`'s
+    documentation for more information
+    """
+    assert len(modpath) > 0
+    if context is not None:
+        try:
+            mtype, mp_filename = _module_file(modpath, [context])
+        except ImportError:
+            mtype, mp_filename = _module_file(modpath, path)
+    else:
+        mtype, mp_filename = _module_file(modpath, path)
+    if mtype == PY_COMPILED:
+        try:
+            return get_source_file(mp_filename)
+        except NoSourceFile:
+            return mp_filename
+    elif mtype == C_BUILTIN:
+        # integrated builtin module
+        return None
+    elif mtype == PKG_DIRECTORY:
+        mp_filename = _has_init(mp_filename)
+    return mp_filename
+
+def _search_zip(modpath, pic):
+    for filepath, importer in pic.items():
+        if importer is not None:
+            if importer.find_module(modpath[0]):
+                if not importer.find_module('/'.join(modpath)):
+                    raise ImportError('No module named %s in %s/%s' % (
+                        '.'.join(modpath[1:]), file, modpath))
+                return ZIPFILE, abspath(filepath) + '/' + '/'.join(modpath), filepath
+    raise ImportError('No module named %s' % '.'.join(modpath))
+
+def _module_file(modpath, path=None):
+    """get a module type / file path
+
+    :type modpath: list or tuple
+    :param modpath:
+      splitted module's name (i.e name of a module or package splitted
+      on '.'), with leading empty strings for explicit relative import
+
+    :type path: list or None
+    :param path:
+      optional list of path where the module or package should be
+      searched (use sys.path if nothing or None is given)
+
+
+    :rtype: tuple(int, str)
+    :return: the module type flag and the file path for a module
+    """
+    # egg support compat
+    try:
+        pic = sys.path_importer_cache
+        _path = (path is None and sys.path or path)
+        for __path in _path:
+            if not __path in pic:
+                try:
+                    pic[__path] = zipimport.zipimporter(__path)
+                except zipimport.ZipImportError:
+                    pic[__path] = None
+        checkeggs = True
+    except AttributeError:
+        checkeggs = False
+    imported = []
+    while modpath:
+        try:
+            _, mp_filename, mp_desc = find_module(modpath[0], path)
+        except ImportError:
+            if checkeggs:
+                return _search_zip(modpath, pic)[:2]
+            raise
+        else:
+            if checkeggs:
+                fullabspath = [abspath(x) for x in _path]
+                try:
+                    pathindex = fullabspath.index(dirname(abspath(mp_filename)))
+                    emtype, emp_filename, zippath = _search_zip(modpath, pic)
+                    if pathindex > _path.index(zippath):
+                        # an egg takes priority
+                        return emtype, emp_filename
+                except ValueError:
+                    # XXX not in _path
+                    pass
+                except ImportError:
+                    pass
+                checkeggs = False
+        imported.append(modpath.pop(0))
+        mtype = mp_desc[2]
+        if modpath:
+            if mtype != PKG_DIRECTORY:
+                raise ImportError('No module %s in %s' % ('.'.join(modpath),
+                                                          '.'.join(imported)))
+            path = [mp_filename]
+    return mtype, mp_filename
+
+def _is_python_file(filename):
+    """return true if the given filename should be considered as a python file
+
+    .pyc and .pyo are ignored
+    """
+    for ext in ('.py', '.so', '.pyd', '.pyw'):
+        if filename.endswith(ext):
+            return True
+    return False
+
+
+def _has_init(directory):
+    """if the given directory has a valid __init__ file, return its path,
+    else return None
+    """
+    mod_or_pack = join(directory, '__init__')
+    for ext in PY_SOURCE_EXTS + ('pyc', 'pyo'):
+        if exists(mod_or_pack + '.' + ext):
+            return mod_or_pack + '.' + ext
+    return None

pylibs/logilab/common/optik_ext.py

@@ -0,0 +1,397 @@
+# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+#
+# This file is part of logilab-common.
+#
+# logilab-common is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation, either version 2.1 of the License, or (at your option) any
+# later version.
+#
+# logilab-common is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with logilab-common.  If not, see <http://www.gnu.org/licenses/>.
+"""Add an abstraction level to transparently import optik classes from optparse
+(python >= 2.3) or the optik package.
+
+It also defines three new types for optik/optparse command line parser :
+
+  * regexp
+    argument of this type will be converted using re.compile
+  * csv
+    argument of this type will be converted using split(',')
+  * yn
+    argument of this type will be true if 'y' or 'yes', false if 'n' or 'no'
+  * named
+    argument of this type are in the form <NAME>=<VALUE> or <NAME>:<VALUE>
+  * password
+    argument of this type wont be converted but this is used by other tools
+    such as interactive prompt for configuration to double check value and
+    use an invisible field
+  * multiple_choice
+    same as default "choice" type but multiple choices allowed
+  * file
+    argument of this type wont be converted but checked that the given file exists
+  * color
+    argument of this type wont be converted but checked its either a
+    named color or a color specified using hexadecimal notation (preceded by a #)
+  * time
+    argument of this type will be converted to a float value in seconds
+    according to time units (ms, s, min, h, d)
+  * bytes
+    argument of this type will be converted to a float value in bytes
+    according to byte units (b, kb, mb, gb, tb)
+"""
+__docformat__ = "restructuredtext en"
+
+import re
+import sys
+import time
+from copy import copy
+from os.path import exists
+
+# python >= 2.3
+from optparse import OptionParser as BaseParser, Option as BaseOption, \
+     OptionGroup, OptionContainer, OptionValueError, OptionError, \
+     Values, HelpFormatter, NO_DEFAULT, SUPPRESS_HELP
+
+try:
+    from mx import DateTime
+    HAS_MX_DATETIME = True
+except ImportError:
+    HAS_MX_DATETIME = False
+
+
+OPTPARSE_FORMAT_DEFAULT = sys.version_info >= (2, 4)
+
+from logilab.common.textutils import splitstrip
+
+def check_regexp(option, opt, value):
+    """check a regexp value by trying to compile it
+    return the compiled regexp
+    """
+    if hasattr(value, 'pattern'):
+        return value
+    try:
+        return re.compile(value)
+    except ValueError:
+        raise OptionValueError(
+            "option %s: invalid regexp value: %r" % (opt, value))
+
+def check_csv(option, opt, value):
+    """check a csv value by trying to split it
+    return the list of separated values
+    """
+    if isinstance(value, (list, tuple)):
+        return value
+    try:
+        return splitstrip(value)
+    except ValueError:
+        raise OptionValueError(
+            "option %s: invalid csv value: %r" % (opt, value))
+
+def check_yn(option, opt, value):
+    """check a yn value
+    return true for yes and false for no
+    """
+    if isinstance(value, int):
+        return bool(value)
+    if value in ('y', 'yes'):
+        return True
+    if value in ('n', 'no'):
+        return False
+    msg = "option %s: invalid yn value %r, should be in (y, yes, n, no)"
+    raise OptionValueError(msg % (opt, value))
+
+def check_named(option, opt, value):
+    """check a named value
+    return a dictionary containing (name, value) associations
+    """
+    if isinstance(value, dict):
+        return value
+    values = []
+    for value in check_csv(option, opt, value):
+        if value.find('=') != -1:
+            values.append(value.split('=', 1))
+        elif value.find(':') != -1:
+            values.append(value.split(':', 1))
+    if values:
+        return dict(values)
+    msg = "option %s: invalid named value %r, should be <NAME>=<VALUE> or \
+<NAME>:<VALUE>"
+    raise OptionValueError(msg % (opt, value))
+
+def check_password(option, opt, value):
+    """check a password value (can't be empty)
+    """
+    # no actual checking, monkey patch if you want more
+    return value
+
+def check_file(option, opt, value):
+    """check a file value
+    return the filepath
+    """
+    if exists(value):
+        return value
+    msg = "option %s: file %r does not exist"
+    raise OptionValueError(msg % (opt, value))
+
+# XXX use python datetime
+def check_date(option, opt, value):
+    """check a file value
+    return the filepath
+    """
+    try:
+        return DateTime.strptime(value, "%Y/%m/%d")
+    except DateTime.Error :
+        raise OptionValueError(
+            "expected format of %s is yyyy/mm/dd" % opt)
+
+def check_color(option, opt, value):
+    """check a color value and returns it
+    /!\ does *not* check color labels (like 'red', 'green'), only
+    checks hexadecimal forms
+    """
+    # Case (1) : color label, we trust the end-user
+    if re.match('[a-z0-9 ]+$', value, re.I):
+        return value
+    # Case (2) : only accepts hexadecimal forms
+    if re.match('#[a-f0-9]{6}', value, re.I):
+        return value
+    # Else : not a color label neither a valid hexadecimal form => error
+    msg = "option %s: invalid color : %r, should be either hexadecimal \
+    value or predefined color"
+    raise OptionValueError(msg % (opt, value))
+
+def check_time(option, opt, value):
+    from logilab.common.textutils import TIME_UNITS, apply_units
+    if isinstance(value, (int, long, float)):
+        return value
+    return apply_units(value, TIME_UNITS)
+
+def check_bytes(option, opt, value):
+    from logilab.common.textutils import BYTE_UNITS, apply_units
+    if hasattr(value, '__int__'):
+        return value
+    return apply_units(value, BYTE_UNITS)
+
+import types
+
+class Option(BaseOption):
+    """override optik.Option to add some new option types
+    """
+    TYPES = BaseOption.TYPES + ('regexp', 'csv', 'yn', 'named', 'password',
+                                'multiple_choice', 'file', 'color',
+                                'time', 'bytes')
+    ATTRS = BaseOption.ATTRS + ['hide', 'level']
+    TYPE_CHECKER = copy(BaseOption.TYPE_CHECKER)
+    TYPE_CHECKER['regexp'] = check_regexp
+    TYPE_CHECKER['csv'] = check_csv
+    TYPE_CHECKER['yn'] = check_yn
+    TYPE_CHECKER['named'] = check_named
+    TYPE_CHECKER['multiple_choice'] = check_csv
+    TYPE_CHECKER['file'] = check_file
+    TYPE_CHECKER['color'] = check_color
+    TYPE_CHECKER['password'] = check_password
+    TYPE_CHECKER['time'] = check_time
+    TYPE_CHECKER['bytes'] = check_bytes
+    if HAS_MX_DATETIME:
+        TYPES += ('date',)
+        TYPE_CHECKER['date'] = check_date
+
+    def __init__(self, *opts, **attrs):
+        BaseOption.__init__(self, *opts, **attrs)
+        if hasattr(self, "hide") and self.hide:
+            self.help = SUPPRESS_HELP
+
+    def _check_choice(self):
+        """FIXME: need to override this due to optik misdesign"""
+        if self.type in ("choice", "multiple_choice"):
+            if self.choices is None:
+                raise OptionError(
+                    "must supply a list of choices for type 'choice'", self)
+            elif type(self.choices) not in (types.TupleType, types.ListType):
+                raise OptionError(
+                    "choices must be a list of strings ('%s' supplied)"
+                    % str(type(self.choices)).split("'")[1], self)
+        elif self.choices is not None:
+            raise OptionError(
+                "must not supply choices for type %r" % self.type, self)
+    BaseOption.CHECK_METHODS[2] = _check_choice
+
+
+    def process(self, opt, value, values, parser):
+        # First, convert the value(s) to the right type.  Howl if any
+        # value(s) are bogus.
+        try:
+            value = self.convert_value(opt, value)
+        except AttributeError: # py < 2.4
+            value = self.check_value(opt, value)
+        if self.type == 'named':
+            existant = getattr(values, self.dest)
+            if existant:
+                existant.update(value)
+                value = existant
+       # And then take whatever action is expected of us.
+        # This is a separate method to make life easier for
+        # subclasses to add new actions.
+        return self.take_action(
+            self.action, self.dest, opt, value, values, parser)
+
+
+class OptionParser(BaseParser):
+    """override optik.OptionParser to use our Option class
+    """
+    def __init__(self, option_class=Option, *args, **kwargs):
+        BaseParser.__init__(self, option_class=Option, *args, **kwargs)
+
+    def format_option_help(self, formatter=None):
+        if formatter is None:
+            formatter = self.formatter
+        outputlevel = getattr(formatter, 'output_level', 0)
+        formatter.store_option_strings(self)
+        result = []
+        result.append(formatter.format_heading("Options"))
+        formatter.indent()
+        if self.option_list:
+            result.append(OptionContainer.format_option_help(self, formatter))
+            result.append("\n")
+        for group in self.option_groups:
+            if group.level <= outputlevel and (
+                group.description or level_options(group, outputlevel)):
+                result.append(group.format_help(formatter))
+                result.append("\n")
+        formatter.dedent()
+        # Drop the last "\n", or the header if no options or option groups:
+        return "".join(result[:-1])
+
+
+OptionGroup.level = 0
+
+def level_options(group, outputlevel):
+    return [option for option in group.option_list
+            if (getattr(option, 'level', 0) or 0) <= outputlevel
+            and not option.help is SUPPRESS_HELP]
+
+def format_option_help(self, formatter):
+    result = []
+    outputlevel = getattr(formatter, 'output_level', 0) or 0
+    for option in level_options(self, outputlevel):
+        result.append(formatter.format_option(option))
+    return "".join(result)
+OptionContainer.format_option_help = format_option_help
+
+
+class ManHelpFormatter(HelpFormatter):
+    """Format help using man pages ROFF format"""
+
+    def __init__ (self,
+                  indent_increment=0,
+                  max_help_position=24,
+                  width=79,
+                  short_first=0):
+        HelpFormatter.__init__ (
+            self, indent_increment, max_help_position, width, short_first)
+
+    def format_heading(self, heading):
+        return '.SH %s\n' % heading.upper()
+
+    def format_description(self, description):
+        return description
+
+    def format_option(self, option):
+        try:
+            optstring = option.option_strings
+        except AttributeError:
+            optstring = self.format_option_strings(option)
+        if option.help:
+            help_text = self.expand_default(option)
+            help = ' '.join([l.strip() for l in help_text.splitlines()])
+        else:
+            help = ''
+        return '''.IP "%s"
+%s
+''' % (optstring, help)
+
+    def format_head(self, optparser, pkginfo, section=1):
+        long_desc = ""
+        try:
+            pgm = optparser._get_prog_name()
+        except AttributeError:
+            # py >= 2.4.X (dunno which X exactly, at least 2)
+            pgm = optparser.get_prog_name()
+        short_desc = self.format_short_description(pgm, pkginfo.description)
+        if hasattr(pkginfo, "long_desc"):
+            long_desc = self.format_long_description(pgm, pkginfo.long_desc)
+        return '%s\n%s\n%s\n%s' % (self.format_title(pgm, section),
+                                   short_desc, self.format_synopsis(pgm),
+                                   long_desc)
+
+    def format_title(self, pgm, section):
+        date = '-'.join([str(num) for num in time.localtime()[:3]])
+        return '.TH %s %s "%s" %s' % (pgm, section, date, pgm)
+
+    def format_short_description(self, pgm, short_desc):
+        return '''.SH NAME
+.B %s
+\- %s
+''' % (pgm, short_desc.strip())
+
+    def format_synopsis(self, pgm):
+        return '''.SH SYNOPSIS
+.B  %s
+[
+.I OPTIONS
+] [
+.I <arguments>
+]
+''' % pgm
+
+    def format_long_description(self, pgm, long_desc):
+        long_desc = '\n'.join([line.lstrip()
+                               for line in long_desc.splitlines()])
+        long_desc = long_desc.replace('\n.\n', '\n\n')
+        if long_desc.lower().startswith(pgm):
+            long_desc = long_desc[len(pgm):]
+        return '''.SH DESCRIPTION
+.B %s
+%s
+''' % (pgm, long_desc.strip())
+
+    def format_tail(self, pkginfo):
+        tail = '''.SH SEE ALSO
+/usr/share/doc/pythonX.Y-%s/
+
+.SH BUGS
+Please report bugs on the project\'s mailing list:
+%s
+
+.SH AUTHOR
+%s <%s>
+''' % (getattr(pkginfo, 'debian_name', pkginfo.modname),
+       pkginfo.mailinglist, pkginfo.author, pkginfo.author_email)
+
+        if hasattr(pkginfo, "copyright"):
+            tail += '''
+.SH COPYRIGHT
+%s
+''' % pkginfo.copyright
+
+        return tail
+
+def generate_manpage(optparser, pkginfo, section=1, stream=sys.stdout, level=0):
+    """generate a man page from an optik parser"""
+    formatter = ManHelpFormatter()
+    formatter.output_level = level
+    formatter.parser = optparser
+    print >> stream, formatter.format_head(optparser, pkginfo, section)
+    print >> stream, optparser.format_option_help(formatter)
+    print >> stream, formatter.format_tail(pkginfo)
+
+
+__all__ = ('OptionParser', 'Option', 'OptionGroup', 'OptionValueError',
+           'Values')

pylibs/logilab/common/optparser.py

@@ -0,0 +1,90 @@
+# -*- coding: utf-8 -*-
+# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+#
+# This file is part of logilab-common.
+#
+# logilab-common is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation, either version 2.1 of the License, or (at your option) any
+# later version.
+#
+# logilab-common is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with logilab-common.  If not, see <http://www.gnu.org/licenses/>.
+"""Extend OptionParser with commands.
+
+Example:
+
+>>> parser = OptionParser()
+>>> parser.usage = '%prog COMMAND [options] <arg> ...'
+>>> parser.add_command('build', 'mymod.build')
+>>> parser.add_command('clean', run_clean, add_opt_clean)
+>>> run, options, args = parser.parse_command(sys.argv[1:])
+>>> return run(options, args[1:])
+
+With mymod.build that defines two functions run and add_options
+"""
+__docformat__ = "restructuredtext en"
+
+from warnings import warn
+warn('lgc.optparser module is deprecated, use lgc.clcommands instead', DeprecationWarning,
+     stacklevel=2)
+
+import sys
+import optparse
+
+class OptionParser(optparse.OptionParser):
+
+    def __init__(self, *args, **kwargs):
+        optparse.OptionParser.__init__(self, *args, **kwargs)
+        self._commands = {}
+        self.min_args, self.max_args = 0, 1
+
+    def add_command(self, name, mod_or_funcs, help=''):
+        """name of the command, name of module or tuple of functions
+        (run, add_options)
+        """
+        assert isinstance(mod_or_funcs, str) or isinstance(mod_or_funcs, tuple), \
+            "mod_or_funcs has to be a module name or a tuple of functions"
+        self._commands[name] = (mod_or_funcs, help)
+
+    def print_main_help(self):
+        optparse.OptionParser.print_help(self)
+        print '\ncommands:'
+        for cmdname, (_, help) in self._commands.items():
+            print '% 10s - %s' % (cmdname, help)
+
+    def parse_command(self, args):
+        if len(args) == 0:
+            self.print_main_help()
+            sys.exit(1)
+        cmd = args[0]
+        args = args[1:]
+        if cmd not in self._commands:
+            if cmd in ('-h', '--help'):
+                self.print_main_help()
+                sys.exit(0)
+            elif self.version is not None and cmd == "--version":
+                self.print_version()
+                sys.exit(0)
+            self.error('unknown command')
+        self.prog = '%s %s' % (self.prog, cmd)
+        mod_or_f, help = self._commands[cmd]
+        # optparse inserts self.description between usage and options help
+        self.description = help
+        if isinstance(mod_or_f, str):
+            exec 'from %s import run, add_options' % mod_or_f
+        else:
+            run, add_options = mod_or_f
+        add_options(self)
+        (options, args) = self.parse_args(args)
+        if not (self.min_args <= len(args) <= self.max_args):
+            self.error('incorrect number of arguments')
+        return run, options, args
+
+

pylibs/logilab/common/pdf_ext.py

@@ -0,0 +1,111 @@
+# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+#
+# This file is part of logilab-common.
+#
+# logilab-common is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation, either version 2.1 of the License, or (at your option) any
+# later version.
+#
+# logilab-common is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with logilab-common.  If not, see <http://www.gnu.org/licenses/>.
+"""Manipulate pdf and fdf files (pdftk recommended).
+
+Notes regarding pdftk, pdf forms and fdf files (form definition file)
+fields names can be extracted with:
+
+    pdftk orig.pdf generate_fdf output truc.fdf
+
+to merge fdf and pdf:
+
+    pdftk orig.pdf fill_form test.fdf output result.pdf [flatten]
+
+without flatten, one could further edit the resulting form.
+with flatten, everything is turned into text.
+
+
+
+
+"""
+__docformat__ = "restructuredtext en"
+# XXX seems very unix specific
+# TODO: check availability of pdftk at import
+
+
+import os
+
+HEAD="""%FDF-1.2
+%\xE2\xE3\xCF\xD3
+1 0 obj
+<<
+/FDF
+<<
+/Fields [
+"""
+
+TAIL="""]
+>>
+>>
+endobj
+trailer
+
+<<
+/Root 1 0 R
+>>
+%%EOF
+"""
+
+def output_field( f ):
+    return "\xfe\xff" + "".join( [ "\x00"+c for c in f ] )
+
+def extract_keys(lines):
+    keys = []
+    for line in lines:
+        if line.startswith('/V'):
+            pass #print 'value',line
+        elif line.startswith('/T'):
+            key = line[7:-2]
+            key = ''.join(key.split('\x00'))
+            keys.append( key )
+    return keys
+
+def write_field(out, key, value):
+    out.write("<<\n")
+    if value:
+        out.write("/V (%s)\n" %value)
+    else:
+        out.write("/V /\n")
+    out.write("/T (%s)\n" % output_field(key) )
+    out.write(">> \n")
+
+def write_fields(out, fields):
+    out.write(HEAD)
+    for (key, value, comment) in fields:
+        write_field(out, key, value)
+        write_field(out, key+"a", value) # pour copie-carbone sur autres pages
+    out.write(TAIL)
+
+def extract_keys_from_pdf(filename):
+    # what about using 'pdftk filename dump_data_fields' and parsing the output ?
+    os.system('pdftk %s generate_fdf output /tmp/toto.fdf' % filename)
+    lines = file('/tmp/toto.fdf').readlines()
+    return extract_keys(lines)
+
+
+def fill_pdf(infile, outfile, fields):
+    write_fields(file('/tmp/toto.fdf', 'w'), fields)
+    os.system('pdftk %s fill_form /tmp/toto.fdf output %s flatten' % (infile, outfile))
+
+def testfill_pdf(infile, outfile):
+    keys = extract_keys_from_pdf(infile)
+    fields = []
+    for key in keys:
+        fields.append( (key, key, '') )
+    fill_pdf(infile, outfile, fields)
+

pylibs/logilab/common/proc.py

@@ -0,0 +1,277 @@
+# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+#
+# This file is part of logilab-common.
+#
+# logilab-common is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation, either version 2.1 of the License, or (at your option) any
+# later version.
+#
+# logilab-common is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with logilab-common.  If not, see <http://www.gnu.org/licenses/>.
+"""module providing:
+* process information (linux specific: rely on /proc)
+* a class for resource control (memory / time / cpu time)
+
+This module doesn't work on windows platforms (only tested on linux)
+
+:organization: Logilab
+
+
+
+"""
+__docformat__ = "restructuredtext en"
+
+import os
+import stat
+from resource import getrlimit, setrlimit, RLIMIT_CPU, RLIMIT_AS
+from signal import signal, SIGXCPU, SIGKILL, SIGUSR2, SIGUSR1
+from threading import Timer, currentThread, Thread, Event
+from time import time
+
+from logilab.common.tree import Node
+
+class NoSuchProcess(Exception): pass
+
+def proc_exists(pid):
+    """check the a pid is registered in /proc
+    raise NoSuchProcess exception if not
+    """
+    if not os.path.exists('/proc/%s' % pid):
+        raise NoSuchProcess()
+
+PPID = 3
+UTIME = 13
+STIME = 14
+CUTIME = 15
+CSTIME = 16
+VSIZE = 22
+
+class ProcInfo(Node):
+    """provide access to process information found in /proc"""
+
+    def __init__(self, pid):
+        self.pid = int(pid)
+        Node.__init__(self, self.pid)
+        proc_exists(self.pid)
+        self.file = '/proc/%s/stat' % self.pid
+        self.ppid = int(self.status()[PPID])
+
+    def memory_usage(self):
+        """return the memory usage of the process in Ko"""
+        try :
+            return int(self.status()[VSIZE])
+        except IOError:
+            return 0
+
+    def lineage_memory_usage(self):
+        return self.memory_usage() + sum([child.lineage_memory_usage()
+                                          for child in self.children])
+
+    def time(self, children=0):
+        """return the number of jiffies that this process has been scheduled
+        in user and kernel mode"""
+        status = self.status()
+        time = int(status[UTIME]) + int(status[STIME])
+        if children:
+            time += int(status[CUTIME]) + int(status[CSTIME])
+        return time
+
+    def status(self):
+        """return the list of fields found in /proc/<pid>/stat"""
+        return open(self.file).read().split()
+
+    def name(self):
+        """return the process name found in /proc/<pid>/stat
+        """
+        return self.status()[1].strip('()')
+
+    def age(self):
+        """return the age of the process
+        """
+        return os.stat(self.file)[stat.ST_MTIME]
+
+class ProcInfoLoader:
+    """manage process information"""
+
+    def __init__(self):
+        self._loaded = {}
+
+    def list_pids(self):
+        """return a list of existent process ids"""
+        for subdir in os.listdir('/proc'):
+            if subdir.isdigit():
+                yield int(subdir)
+
+    def load(self, pid):
+        """get a ProcInfo object for a given pid"""
+        pid = int(pid)
+        try:
+            return self._loaded[pid]
+        except KeyError:
+            procinfo = ProcInfo(pid)
+            procinfo.manager = self
+            self._loaded[pid] = procinfo
+            return procinfo
+
+
+    def load_all(self):
+        """load all processes information"""
+        for pid in self.list_pids():
+            try:
+                procinfo = self.load(pid)
+                if procinfo.parent is None and procinfo.ppid:
+                    pprocinfo = self.load(procinfo.ppid)
+                    pprocinfo.append(procinfo)
+            except NoSuchProcess:
+                pass
+
+
+try:
+    class ResourceError(BaseException):
+        """Error raise when resource limit is reached"""
+        limit = "Unknown Resource Limit"
+except NameError:
+    class ResourceError(Exception):
+        """Error raise when resource limit is reached"""
+        limit = "Unknown Resource Limit"
+
+
+class XCPUError(ResourceError):
+    """Error raised when CPU Time limit is reached"""
+    limit = "CPU Time"
+
+class LineageMemoryError(ResourceError):
+    """Error raised when the total amount of memory used by a process and
+    it's child is reached"""
+    limit = "Lineage total Memory"
+
+class TimeoutError(ResourceError):
+    """Error raised when the process is running for to much time"""
+    limit = "Real Time"
+
+# Can't use subclass because the StandardError MemoryError raised
+RESOURCE_LIMIT_EXCEPTION = (ResourceError, MemoryError)
+
+
+class MemorySentinel(Thread):
+    """A class checking a process don't use too much memory in a separated
+    daemonic thread
+    """
+    def __init__(self, interval, memory_limit, gpid=os.getpid()):
+        Thread.__init__(self, target=self._run, name="Test.Sentinel")
+        self.memory_limit = memory_limit
+        self._stop = Event()
+        self.interval = interval
+        self.setDaemon(True)
+        self.gpid = gpid
+
+    def stop(self):
+        """stop ap"""
+        self._stop.set()
+
+    def _run(self):
+        pil = ProcInfoLoader()
+        while not self._stop.isSet():
+            if self.memory_limit <= pil.load(self.gpid).lineage_memory_usage():
+                os.killpg(self.gpid, SIGUSR1)
+            self._stop.wait(self.interval)
+
+
+class ResourceController:
+
+    def __init__(self, max_cpu_time=None, max_time=None, max_memory=None,
+                 max_reprieve=60):
+        if SIGXCPU == -1:
+            raise RuntimeError("Unsupported platform")
+        self.max_time = max_time
+        self.max_memory = max_memory
+        self.max_cpu_time = max_cpu_time
+        self._reprieve = max_reprieve
+        self._timer = None
+        self._msentinel = None
+        self._old_max_memory = None
+        self._old_usr1_hdlr = None
+        self._old_max_cpu_time = None
+        self._old_usr2_hdlr = None
+        self._old_sigxcpu_hdlr = None
+        self._limit_set = 0
+        self._abort_try = 0
+        self._start_time = None
+        self._elapse_time = 0
+
+    def _hangle_sig_timeout(self, sig, frame):
+        raise TimeoutError()
+
+    def _hangle_sig_memory(self, sig, frame):
+        if self._abort_try < self._reprieve:
+            self._abort_try += 1
+            raise LineageMemoryError("Memory limit reached")
+        else:
+            os.killpg(os.getpid(), SIGKILL)
+
+    def _handle_sigxcpu(self, sig, frame):
+        if self._abort_try < self._reprieve:
+            self._abort_try += 1
+            raise XCPUError("Soft CPU time limit reached")
+        else:
+            os.killpg(os.getpid(), SIGKILL)
+
+    def _time_out(self):
+        if self._abort_try < self._reprieve:
+            self._abort_try += 1
+            os.killpg(os.getpid(), SIGUSR2)
+            if self._limit_set > 0:
+                self._timer = Timer(1, self._time_out)
+                self._timer.start()
+        else:
+            os.killpg(os.getpid(), SIGKILL)
+
+    def setup_limit(self):
+        """set up the process limit"""
+        assert currentThread().getName() == 'MainThread'
+        os.setpgrp()
+        if self._limit_set <= 0:
+            if self.max_time is not None:
+                self._old_usr2_hdlr = signal(SIGUSR2, self._hangle_sig_timeout)
+                self._timer = Timer(max(1, int(self.max_time) - self._elapse_time),
+                                    self._time_out)
+                self._start_time = int(time())
+                self._timer.start()
+            if self.max_cpu_time is not None:
+                self._old_max_cpu_time = getrlimit(RLIMIT_CPU)
+                cpu_limit = (int(self.max_cpu_time), self._old_max_cpu_time[1])
+                self._old_sigxcpu_hdlr = signal(SIGXCPU, self._handle_sigxcpu)
+                setrlimit(RLIMIT_CPU, cpu_limit)
+            if self.max_memory is not None:
+                self._msentinel = MemorySentinel(1, int(self.max_memory) )
+                self._old_max_memory = getrlimit(RLIMIT_AS)
+                self._old_usr1_hdlr = signal(SIGUSR1, self._hangle_sig_memory)
+                as_limit = (int(self.max_memory), self._old_max_memory[1])
+                setrlimit(RLIMIT_AS, as_limit)
+                self._msentinel.start()
+        self._limit_set += 1
+
+    def clean_limit(self):
+        """reinstall the old process limit"""
+        if self._limit_set > 0:
+            if self.max_time is not None:
+                self._timer.cancel()
+                self._elapse_time += int(time())-self._start_time
+                self._timer = None
+                signal(SIGUSR2, self._old_usr2_hdlr)
+            if self.max_cpu_time is not None:
+                setrlimit(RLIMIT_CPU, self._old_max_cpu_time)
+                signal(SIGXCPU, self._old_sigxcpu_hdlr)
+            if self.max_memory is not None:
+                self._msentinel.stop()
+                self._msentinel = None
+                setrlimit(RLIMIT_AS, self._old_max_memory)
+                signal(SIGUSR1, self._old_usr1_hdlr)
+        self._limit_set -= 1

pylibs/logilab/common/pyro_ext.py

@@ -0,0 +1,180 @@
+# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+#
+# This file is part of logilab-common.
+#
+# logilab-common is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation, either version 2.1 of the License, or (at your option) any
+# later version.
+#
+# logilab-common is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with logilab-common.  If not, see <http://www.gnu.org/licenses/>.
+"""Python Remote Object utilities
+
+Main functions available:
+
+* `register_object` to expose arbitrary object through pyro using delegation
+  approach and register it in the nameserver.
+* `ns_unregister` unregister an object identifier from the nameserver.
+* `ns_get_proxy` get a pyro proxy from a nameserver object identifier.
+"""
+
+__docformat__ = "restructuredtext en"
+
+import logging
+import tempfile
+
+from Pyro import core, naming, errors, util, config
+
+_LOGGER = logging.getLogger('pyro')
+_MARKER = object()
+
+config.PYRO_STORAGE = tempfile.gettempdir()
+
+def ns_group_and_id(idstr, defaultnsgroup=_MARKER):
+    try:
+        nsgroup, nsid = idstr.rsplit('.', 1)
+    except ValueError:
+        if defaultnsgroup is _MARKER:
+            nsgroup = config.PYRO_NS_DEFAULTGROUP
+        else:
+            nsgroup = defaultnsgroup
+        nsid = idstr
+    if nsgroup is not None and not nsgroup.startswith(':'):
+        nsgroup = ':' + nsgroup
+    return nsgroup, nsid
+
+def host_and_port(hoststr):
+    if not hoststr:
+        return None, None
+    try:
+        hoststr, port = hoststr.split(':')
+    except ValueError:
+        port = None
+    else:
+        port = int(port)
+    return hoststr, port
+
+_DAEMONS = {}
+_PYRO_OBJS = {}
+def _get_daemon(daemonhost, start=True):
+    if not daemonhost in _DAEMONS:
+        if not start:
+            raise Exception('no daemon for %s' % daemonhost)
+        if not _DAEMONS:
+            core.initServer(banner=0)
+        host, port = host_and_port(daemonhost)
+        daemon = core.Daemon(host=host, port=port)
+        _DAEMONS[daemonhost] = daemon
+    return _DAEMONS[daemonhost]
+
+
+def locate_ns(nshost):
+    """locate and return the pyro name server to the daemon"""
+    core.initClient(banner=False)
+    return naming.NameServerLocator().getNS(*host_and_port(nshost))
+
+
+def register_object(object, nsid, defaultnsgroup=_MARKER,
+                    daemonhost=None, nshost=None, use_pyrons=True):
+    """expose the object as a pyro object and register it in the name-server
+
+    if use_pyrons is False, then the object is exposed, but no
+    attempt to register it to a pyro nameserver is made.
+
+    return the pyro daemon object
+    """
+    nsgroup, nsid = ns_group_and_id(nsid, defaultnsgroup)
+    daemon = _get_daemon(daemonhost)
+    if use_pyrons:
+        nsd = locate_ns(nshost)
+        # make sure our namespace group exists
+        try:
+            nsd.createGroup(nsgroup)
+        except errors.NamingError:
+            pass
+        daemon.useNameServer(nsd)
+    # use Delegation approach
+    impl = core.ObjBase()
+    impl.delegateTo(object)
+    qnsid = '%s.%s' % (nsgroup, nsid)
+    uri = daemon.connect(impl, qnsid)
+    _PYRO_OBJS[qnsid] = str(uri)
+    _LOGGER.info('registered %s a pyro object using group %s and id %s',
+                 object, nsgroup, nsid)
+    return daemon
+
+def get_object_uri(qnsid):
+    return _PYRO_OBJS[qnsid]
+
+def ns_unregister(nsid, defaultnsgroup=_MARKER, nshost=None):
+    """unregister the object with the given nsid from the pyro name server"""
+    nsgroup, nsid = ns_group_and_id(nsid, defaultnsgroup)
+    try:
+        nsd = locate_ns(nshost)
+    except errors.PyroError, ex:
+        # name server not responding
+        _LOGGER.error('can\'t locate pyro name server: %s', ex)
+    else:
+        try:
+            nsd.unregister('%s.%s' % (nsgroup, nsid))
+            _LOGGER.info('%s unregistered from pyro name server', nsid)
+        except errors.NamingError:
+            _LOGGER.warning('%s not registered in pyro name server', nsid)
+
+
+def ns_reregister(nsid, defaultnsgroup=_MARKER, nshost=None):
+    """reregister a pyro object into the name server. You only have to specify
+    the name-server id of the object (though you MUST have gone through
+    `register_object` for the given object previously).
+
+    This is especially useful for long running server while the name server may
+    have been restarted, and its records lost.
+    """
+    nsgroup, nsid = ns_group_and_id(nsid, defaultnsgroup)
+    qnsid = '%s.%s' % (nsgroup, nsid)
+    nsd = locate_ns(nshost)
+    try:
+        nsd.unregister(qnsid)
+    except errors.NamingError:
+        # make sure our namespace group exists
+        try:
+            nsd.createGroup(nsgroup)
+        except errors.NamingError:
+            pass
+    nsd.register(qnsid, _PYRO_OBJS[qnsid])
+
+def ns_get_proxy(nsid, defaultnsgroup=_MARKER, nshost=None):
+    """
+    if nshost is None, the nameserver is found by a broadcast.
+    """
+    # resolve the Pyro object
+    nsgroup, nsid = ns_group_and_id(nsid, defaultnsgroup)
+    try:
+        nsd = locate_ns(nshost)
+        pyrouri = nsd.resolve('%s.%s' % (nsgroup, nsid))
+    except errors.ProtocolError, ex:
+        raise errors.PyroError(
+            'Could not connect to the Pyro name server (host: %s)' % nshost)
+    except errors.NamingError:
+        raise errors.PyroError(
+            'Could not get proxy for %s (not registered in Pyro), '
+            'you may have to restart your server-side application' % nsid)
+    return core.getProxyForURI(pyrouri)
+
+def get_proxy(pyro_uri):
+    """get a proxy for the passed pyro uri without using a nameserver
+    """
+    return core.getProxyForURI(pyro_uri)
+
+def set_pyro_log_threshold(level):
+    pyrologger = logging.getLogger('Pyro.%s' % str(id(util.Log)))
+    # remove handlers so only the root handler is used
+    pyrologger.handlers = []
+    pyrologger.setLevel(level)

pylibs/logilab/common/pytest.py

@@ -0,0 +1,1177 @@
+# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+#
+# This file is part of logilab-common.
+#
+# logilab-common is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation, either version 2.1 of the License, or (at your option) any
+# later version.
+#
+# logilab-common is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with logilab-common.  If not, see <http://www.gnu.org/licenses/>.
+"""pytest is a tool that eases test running and debugging.
+
+To be able to use pytest, you should either write tests using
+the logilab.common.testlib's framework or the unittest module of the
+Python's standard library.
+
+You can customize pytest's behaviour by defining a ``pytestconf.py`` file
+somewhere in your test directory. In this file, you can add options or
+change the way tests are run.
+
+To add command line options, you must define a ``update_parser`` function in
+your ``pytestconf.py`` file. The function must accept a single parameter
+that will be the OptionParser's instance to customize.
+
+If you wish to customize the tester, you'll have to define a class named
+``CustomPyTester``. This class should extend the default `PyTester` class
+defined in the pytest module. Take a look at the `PyTester` and `DjangoTester`
+classes for more information about what can be done.
+
+For instance, if you wish to add a custom -l option to specify a loglevel, you
+could define the following ``pytestconf.py`` file ::
+
+    import logging
+    from logilab.common.pytest import PyTester
+
+    def update_parser(parser):
+        parser.add_option('-l', '--loglevel', dest='loglevel', action='store',
+                          choices=('debug', 'info', 'warning', 'error', 'critical'),
+                          default='critical', help="the default log level possible choices are "
+                          "('debug', 'info', 'warning', 'error', 'critical')")
+        return parser
+
+
+    class CustomPyTester(PyTester):
+        def __init__(self, cvg, options):
+            super(CustomPyTester, self).__init__(cvg, options)
+            loglevel = options.loglevel.upper()
+            logger = logging.getLogger('erudi')
+            logger.setLevel(logging.getLevelName(loglevel))
+
+
+In your TestCase class you can then get the value of a specific option with
+the ``optval`` method::
+
+    class MyTestCase(TestCase):
+        def test_foo(self):
+            loglevel = self.optval('loglevel')
+            # ...
+
+
+You can also tag your tag your test for fine filtering
+
+With those tag::
+
+    from logilab.common.testlib import tag, TestCase
+
+    class Exemple(TestCase):
+
+        @tag('rouge', 'carre')
+        def toto(self):
+            pass
+
+        @tag('carre', 'vert')
+        def tata(self):
+            pass
+
+        @tag('rouge')
+        def titi(test):
+            pass
+
+you can filter the function with a simple python expression
+
+ * ``toto`` and ``titi`` match ``rouge``
+ * ``toto``, ``tata`` and ``titi``, match ``rouge or carre``
+ * ``tata`` and ``titi`` match``rouge ^ carre``
+ * ``titi`` match ``rouge and not carre``
+"""
+__docformat__ = "restructuredtext en"
+
+PYTEST_DOC = """%prog [OPTIONS] [testfile [testpattern]]
+
+examples:
+
+pytest path/to/mytests.py
+pytest path/to/mytests.py TheseTests
+pytest path/to/mytests.py TheseTests.test_thisone
+pytest path/to/mytests.py -m '(not long and database) or regr'
+
+pytest one (will run both test_thisone and test_thatone)
+pytest path/to/mytests.py -s not (will skip test_notthisone)
+
+pytest --coverage test_foo.py
+  (only if logilab.devtools is available)
+"""
+
+ENABLE_DBC = False
+FILE_RESTART = ".pytest.restart"
+
+import os, sys, re
+import os.path as osp
+from time import time, clock
+import warnings
+import types
+
+from logilab.common.fileutils import abspath_listdir
+from logilab.common import textutils
+from logilab.common import testlib, STD_BLACKLIST
+# use the same unittest module as testlib
+from logilab.common.testlib import unittest, start_interactive_mode
+from logilab.common.compat import any
+import doctest
+
+import unittest as unittest_legacy
+if not getattr(unittest_legacy, "__package__", None):
+    try:
+        import unittest2.suite as unittest_suite
+    except ImportError:
+        sys.exit("You have to install python-unittest2 to use this module")
+else:
+    import unittest.suite as unittest_suite
+
+try:
+    import django
+    from logilab.common.modutils import modpath_from_file, load_module_from_modpath
+    DJANGO_FOUND = True
+except ImportError:
+    DJANGO_FOUND = False
+
+CONF_FILE = 'pytestconf.py'
+
+## coverage hacks, do not read this, do not read this, do not read this
+
+# hey, but this is an aspect, right ?!!!
+class TraceController(object):
+    nesting = 0
+
+    def pause_tracing(cls):
+        if not cls.nesting:
+            cls.tracefunc = staticmethod(getattr(sys, '__settrace__', sys.settrace))
+            cls.oldtracer = getattr(sys, '__tracer__', None)
+            sys.__notrace__ = True
+            cls.tracefunc(None)
+        cls.nesting += 1
+    pause_tracing = classmethod(pause_tracing)
+
+    def resume_tracing(cls):
+        cls.nesting -= 1
+        assert cls.nesting >= 0
+        if not cls.nesting:
+            cls.tracefunc(cls.oldtracer)
+            delattr(sys, '__notrace__')
+    resume_tracing = classmethod(resume_tracing)
+
+
+pause_tracing = TraceController.pause_tracing
+resume_tracing = TraceController.resume_tracing
+
+
+def nocoverage(func):
+    if hasattr(func, 'uncovered'):
+        return func
+    func.uncovered = True
+    def not_covered(*args, **kwargs):
+        pause_tracing()
+        try:
+            return func(*args, **kwargs)
+        finally:
+            resume_tracing()
+    not_covered.uncovered = True
+    return not_covered
+
+
+## end of coverage hacks
+
+
+TESTFILE_RE = re.compile("^((unit)?test.*|smoketest)\.py$")
+def this_is_a_testfile(filename):
+    """returns True if `filename` seems to be a test file"""
+    return TESTFILE_RE.match(osp.basename(filename))
+
+TESTDIR_RE = re.compile("^(unit)?tests?$")
+def this_is_a_testdir(dirpath):
+    """returns True if `filename` seems to be a test directory"""
+    return TESTDIR_RE.match(osp.basename(dirpath))
+
+
+def load_pytest_conf(path, parser):
+    """loads a ``pytestconf.py`` file and update default parser
+    and / or tester.
+    """
+    namespace = {}
+    execfile(path, namespace)
+    if 'update_parser' in namespace:
+        namespace['update_parser'](parser)
+    return namespace.get('CustomPyTester', PyTester)
+
+
+def project_root(parser, projdir=os.getcwd()):
+    """try to find project's root and add it to sys.path"""
+    previousdir = curdir = osp.abspath(projdir)
+    testercls = PyTester
+    conf_file_path = osp.join(curdir, CONF_FILE)
+    if osp.isfile(conf_file_path):
+        testercls = load_pytest_conf(conf_file_path, parser)
+    while this_is_a_testdir(curdir) or \
+              osp.isfile(osp.join(curdir, '__init__.py')):
+        newdir = osp.normpath(osp.join(curdir, os.pardir))
+        if newdir == curdir:
+            break
+        previousdir = curdir
+        curdir = newdir
+        conf_file_path = osp.join(curdir, CONF_FILE)
+        if osp.isfile(conf_file_path):
+            testercls = load_pytest_conf(conf_file_path, parser)
+    return previousdir, testercls
+
+
+class GlobalTestReport(object):
+    """this class holds global test statistics"""
+    def __init__(self):
+        self.ran = 0
+        self.skipped = 0
+        self.failures = 0
+        self.errors = 0
+        self.ttime = 0
+        self.ctime = 0
+        self.modulescount = 0
+        self.errmodules = []
+
+    def feed(self, filename, testresult, ttime, ctime):
+        """integrates new test information into internal statistics"""
+        ran = testresult.testsRun
+        self.ran += ran
+        self.skipped += len(getattr(testresult, 'skipped', ()))
+        self.failures += len(testresult.failures)
+        self.errors += len(testresult.errors)
+        self.ttime += ttime
+        self.ctime += ctime
+        self.modulescount += 1
+        if not testresult.wasSuccessful():
+            problems = len(testresult.failures) + len(testresult.errors)
+            self.errmodules.append((filename[:-3], problems, ran))
+
+    def failed_to_test_module(self, filename):
+        """called when the test module could not be imported by unittest
+        """
+        self.errors += 1
+        self.modulescount += 1
+        self.ran += 1
+        self.errmodules.append((filename[:-3], 1, 1))
+
+    def skip_module(self, filename):
+        self.modulescount += 1
+        self.ran += 1
+        self.errmodules.append((filename[:-3], 0, 0))
+
+    def __str__(self):
+        """this is just presentation stuff"""
+        line1 = ['Ran %s test cases in %.2fs (%.2fs CPU)'
+                 % (self.ran, self.ttime, self.ctime)]
+        if self.errors:
+            line1.append('%s errors' % self.errors)
+        if self.failures:
+            line1.append('%s failures' % self.failures)
+        if self.skipped:
+            line1.append('%s skipped' % self.skipped)
+        modulesok = self.modulescount - len(self.errmodules)
+        if self.errors or self.failures:
+            line2 = '%s modules OK (%s failed)' % (modulesok,
+                                                   len(self.errmodules))
+            descr = ', '.join(['%s [%s/%s]' % info for info in self.errmodules])
+            line3 = '\nfailures: %s' % descr
+        elif modulesok:
+            line2 = 'All %s modules OK' % modulesok
+            line3 = ''
+        else:
+            return ''
+        return '%s\n%s%s' % (', '.join(line1), line2, line3)
+
+
+
+def remove_local_modules_from_sys(testdir):
+    """remove all modules from cache that come from `testdir`
+
+    This is used to avoid strange side-effects when using the
+    testall() mode of pytest.
+    For instance, if we run pytest on this tree::
+
+      A/test/test_utils.py
+      B/test/test_utils.py
+
+    we **have** to clean sys.modules to make sure the correct test_utils
+    module is ran in B
+    """
+    for modname, mod in sys.modules.items():
+        if mod is None:
+            continue
+        if not hasattr(mod, '__file__'):
+            # this is the case of some built-in modules like sys, imp, marshal
+            continue
+        modfile = mod.__file__
+        # if modfile is not an absolute path, it was probably loaded locally
+        # during the tests
+        if not osp.isabs(modfile) or modfile.startswith(testdir):
+            del sys.modules[modname]
+
+
+
+class PyTester(object):
+    """encapsulates testrun logic"""
+
+    def __init__(self, cvg, options):
+        self.report = GlobalTestReport()
+        self.cvg = cvg
+        self.options = options
+        self.firstwrite = True
+        self._errcode = None
+
+    def show_report(self):
+        """prints the report and returns appropriate exitcode"""
+        # everything has been ran, print report
+        print "*" * 79
+        print self.report
+
+    def get_errcode(self):
+        # errcode set explicitly
+        if self._errcode is not None:
+            return self._errcode
+        return self.report.failures + self.report.errors
+
+    def set_errcode(self, errcode):
+        self._errcode = errcode
+    errcode = property(get_errcode, set_errcode)
+
+    def testall(self, exitfirst=False):
+        """walks through current working directory, finds something
+        which can be considered as a testdir and runs every test there
+        """
+        here = os.getcwd()
+        for dirname, dirs, _ in os.walk(here):
+            for skipped in STD_BLACKLIST:
+                if skipped in dirs:
+                    dirs.remove(skipped)
+            basename = osp.basename(dirname)
+            if this_is_a_testdir(basename):
+                print "going into", dirname
+                # we found a testdir, let's explore it !
+                if not self.testonedir(dirname, exitfirst):
+                    break
+                dirs[:] = []
+        if self.report.ran == 0:
+            print "no test dir found testing here:", here
+            # if no test was found during the visit, consider
+            # the local directory as a test directory even if
+            # it doesn't have a traditional test directory name
+            self.testonedir(here)
+
+    def testonedir(self, testdir, exitfirst=False):
+        """finds each testfile in the `testdir` and runs it
+
+        return true when all tests has been executed, false if exitfirst and
+        some test has failed.
+        """
+        for filename in abspath_listdir(testdir):
+            if this_is_a_testfile(filename):
+                if self.options.exitfirst and not self.options.restart:
+                    # overwrite restart file
+                    try:
+                        restartfile = open(FILE_RESTART, "w")
+                        restartfile.close()
+                    except Exception, e:
+                        print >> sys.__stderr__, "Error while overwriting \
+succeeded test file :", osp.join(os.getcwd(), FILE_RESTART)
+                        raise e
+                # run test and collect information
+                prog = self.testfile(filename, batchmode=True)
+                if exitfirst and (prog is None or not prog.result.wasSuccessful()):
+                    return False
+                self.firstwrite = True
+        # clean local modules
+        remove_local_modules_from_sys(testdir)
+        return True
+
+    def testfile(self, filename, batchmode=False):
+        """runs every test in `filename`
+
+        :param filename: an absolute path pointing to a unittest file
+        """
+        here = os.getcwd()
+        dirname = osp.dirname(filename)
+        if dirname:
+            os.chdir(dirname)
+        # overwrite restart file if it has not been done already
+        if self.options.exitfirst and not self.options.restart and self.firstwrite:
+            try:
+                restartfile = open(FILE_RESTART, "w")
+                restartfile.close()
+            except Exception, e:
+                print >> sys.__stderr__, "Error while overwriting \
+succeeded test file :", osp.join(os.getcwd(), FILE_RESTART)
+                raise e
+        modname = osp.basename(filename)[:-3]
+        try:
+            print >> sys.stderr, ('  %s  ' % osp.basename(filename)).center(70, '=')
+        except TypeError: # < py 2.4 bw compat
+            print >> sys.stderr, ('  %s  ' % osp.basename(filename)).center(70)
+        try:
+            tstart, cstart = time(), clock()
+            try:
+                testprog = SkipAwareTestProgram(modname, batchmode=batchmode, cvg=self.cvg,
+                                                 options=self.options, outstream=sys.stderr)
+            except KeyboardInterrupt:
+                raise
+            except SystemExit, exc:
+                self.errcode = exc.code
+                raise
+            except testlib.SkipTest:
+                print "Module skipped:", filename
+                self.report.skip_module(filename)
+                return None
+            except Exception:
+                self.report.failed_to_test_module(filename)
+                print >> sys.stderr, 'unhandled exception occurred while testing', modname
+                import traceback
+                traceback.print_exc(file=sys.stderr)
+                return None
+
+            tend, cend = time(), clock()
+            ttime, ctime = (tend - tstart), (cend - cstart)
+            self.report.feed(filename, testprog.result, ttime, ctime)
+            return testprog
+        finally:
+            if dirname:
+                os.chdir(here)
+
+
+
+class DjangoTester(PyTester):
+
+    def load_django_settings(self, dirname):
+        """try to find project's setting and load it"""
+        curdir = osp.abspath(dirname)
+        previousdir = curdir
+        while not osp.isfile(osp.join(curdir, 'settings.py')) and \
+                  osp.isfile(osp.join(curdir, '__init__.py')):
+            newdir = osp.normpath(osp.join(curdir, os.pardir))
+            if newdir == curdir:
+                raise AssertionError('could not find settings.py')
+            previousdir = curdir
+            curdir = newdir
+        # late django initialization
+        settings = load_module_from_modpath(modpath_from_file(osp.join(curdir, 'settings.py')))
+        from django.core.management import setup_environ
+        setup_environ(settings)
+        settings.DEBUG = False
+        self.settings = settings
+        # add settings dir to pythonpath since it's the project's root
+        if curdir not in sys.path:
+            sys.path.insert(1, curdir)
+
+    def before_testfile(self):
+        # Those imports must be done **after** setup_environ was called
+        from django.test.utils import setup_test_environment
+        from django.test.utils import create_test_db
+        setup_test_environment()
+        create_test_db(verbosity=0)
+        self.dbname = self.settings.TEST_DATABASE_NAME
+
+    def after_testfile(self):
+        # Those imports must be done **after** setup_environ was called
+        from django.test.utils import teardown_test_environment
+        from django.test.utils import destroy_test_db
+        teardown_test_environment()
+        print 'destroying', self.dbname
+        destroy_test_db(self.dbname, verbosity=0)
+
+    def testall(self, exitfirst=False):
+        """walks through current working directory, finds something
+        which can be considered as a testdir and runs every test there
+        """
+        for dirname, dirs, files in os.walk(os.getcwd()):
+            for skipped in ('CVS', '.svn', '.hg'):
+                if skipped in dirs:
+                    dirs.remove(skipped)
+            if 'tests.py' in files:
+                if not self.testonedir(dirname, exitfirst):
+                    break
+                dirs[:] = []
+            else:
+                basename = osp.basename(dirname)
+                if basename in ('test', 'tests'):
+                    print "going into", dirname
+                    # we found a testdir, let's explore it !
+                    if not self.testonedir(dirname, exitfirst):
+                        break
+                    dirs[:] = []
+
+    def testonedir(self, testdir, exitfirst=False):
+        """finds each testfile in the `testdir` and runs it
+
+        return true when all tests has been executed, false if exitfirst and
+        some test has failed.
+        """
+        # special django behaviour : if tests are splitted in several files,
+        # remove the main tests.py file and tests each test file separately
+        testfiles = [fpath for fpath in abspath_listdir(testdir)
+                     if this_is_a_testfile(fpath)]
+        if len(testfiles) > 1:
+            try:
+                testfiles.remove(osp.join(testdir, 'tests.py'))
+            except ValueError:
+                pass
+        for filename in testfiles:
+            # run test and collect information
+            prog = self.testfile(filename, batchmode=True)
+            if exitfirst and (prog is None or not prog.result.wasSuccessful()):
+                return False
+        # clean local modules
+        remove_local_modules_from_sys(testdir)
+        return True
+
+    def testfile(self, filename, batchmode=False):
+        """runs every test in `filename`
+
+        :param filename: an absolute path pointing to a unittest file
+        """
+        here = os.getcwd()
+        dirname = osp.dirname(filename)
+        if dirname:
+            os.chdir(dirname)
+        self.load_django_settings(dirname)
+        modname = osp.basename(filename)[:-3]
+        print >>sys.stderr, ('  %s  ' % osp.basename(filename)).center(70, '=')
+        try:
+            try:
+                tstart, cstart = time(), clock()
+                self.before_testfile()
+                testprog = SkipAwareTestProgram(modname, batchmode=batchmode, cvg=self.cvg)
+                tend, cend = time(), clock()
+                ttime, ctime = (tend - tstart), (cend - cstart)
+                self.report.feed(filename, testprog.result, ttime, ctime)
+                return testprog
+            except SystemExit:
+                raise
+            except Exception, exc:
+                import traceback
+                traceback.print_exc()
+                self.report.failed_to_test_module(filename)
+                print 'unhandled exception occurred while testing', modname
+                print 'error: %s' % exc
+                return None
+        finally:
+            self.after_testfile()
+            if dirname:
+                os.chdir(here)
+
+
+def make_parser():
+    """creates the OptionParser instance
+    """
+    from optparse import OptionParser
+    parser = OptionParser(usage=PYTEST_DOC)
+
+    parser.newargs = []
+    def rebuild_cmdline(option, opt, value, parser):
+        """carry the option to unittest_main"""
+        parser.newargs.append(opt)
+
+    def rebuild_and_store(option, opt, value, parser):
+        """carry the option to unittest_main and store
+        the value on current parser
+        """
+        parser.newargs.append(opt)
+        setattr(parser.values, option.dest, True)
+
+    def capture_and_rebuild(option, opt, value, parser):
+        warnings.simplefilter('ignore', DeprecationWarning)
+        rebuild_cmdline(option, opt, value, parser)
+
+    # pytest options
+    parser.add_option('-t', dest='testdir', default=None,
+                      help="directory where the tests will be found")
+    parser.add_option('-d', dest='dbc', default=False,
+                      action="store_true", help="enable design-by-contract")
+    # unittest_main options provided and passed through pytest
+    parser.add_option('-v', '--verbose', callback=rebuild_cmdline,
+                      action="callback", help="Verbose output")
+    parser.add_option('-i', '--pdb', callback=rebuild_and_store,
+                      dest="pdb", action="callback",
+                      help="Enable test failure inspection (conflicts with --coverage)")
+    parser.add_option('-x', '--exitfirst', callback=rebuild_and_store,
+                      dest="exitfirst", default=False,
+                      action="callback", help="Exit on first failure "
+                      "(only make sense when pytest run one test file)")
+    parser.add_option('-R', '--restart', callback=rebuild_and_store,
+                      dest="restart", default=False,
+                      action="callback",
+                      help="Restart tests from where it failed (implies exitfirst) "
+                        "(only make sense if tests previously ran with exitfirst only)")
+    parser.add_option('--color', callback=rebuild_cmdline,
+                      action="callback",
+                      help="colorize tracebacks")
+    parser.add_option('-s', '--skip',
+                      # XXX: I wish I could use the callback action but it
+                      #      doesn't seem to be able to get the value
+                      #      associated to the option
+                      action="store", dest="skipped", default=None,
+                      help="test names matching this name will be skipped "
+                      "to skip several patterns, use commas")
+    parser.add_option('-q', '--quiet', callback=rebuild_cmdline,
+                      action="callback", help="Minimal output")
+    parser.add_option('-P', '--profile', default=None, dest='profile',
+                      help="Profile execution and store data in the given file")
+    parser.add_option('-m', '--match', default=None, dest='tags_pattern',
+                      help="only execute test whose tag match the current pattern")
+
+    try:
+        from logilab.devtools.lib.coverage import Coverage
+        parser.add_option('--coverage', dest="coverage", default=False,
+                          action="store_true",
+                          help="run tests with pycoverage (conflicts with --pdb)")
+    except ImportError:
+        pass
+
+    if DJANGO_FOUND:
+        parser.add_option('-J', '--django', dest='django', default=False,
+                          action="store_true",
+                          help='use pytest for django test cases')
+    return parser
+
+
+def parseargs(parser):
+    """Parse the command line and return (options processed), (options to pass to
+    unittest_main()), (explicitfile or None).
+    """
+    # parse the command line
+    options, args = parser.parse_args()
+    if options.pdb and getattr(options, 'coverage', False):
+        parser.error("'pdb' and 'coverage' options are exclusive")
+    filenames = [arg for arg in args if arg.endswith('.py')]
+    if filenames:
+        if len(filenames) > 1:
+            parser.error("only one filename is acceptable")
+        explicitfile = filenames[0]
+        args.remove(explicitfile)
+    else:
+        explicitfile = None
+    # someone wants DBC
+    testlib.ENABLE_DBC = options.dbc
+    newargs = parser.newargs
+    if options.skipped:
+        newargs.extend(['--skip', options.skipped])
+    # restart implies exitfirst
+    if options.restart:
+        options.exitfirst = True
+    # append additional args to the new sys.argv and let unittest_main
+    # do the rest
+    newargs += args
+    return options, explicitfile
+
+
+
+def run():
+    parser = make_parser()
+    rootdir, testercls = project_root(parser)
+    options, explicitfile = parseargs(parser)
+    # mock a new command line
+    sys.argv[1:] = parser.newargs
+    covermode = getattr(options, 'coverage', None)
+    cvg = None
+    if not '' in sys.path:
+        sys.path.insert(0, '')
+    if covermode:
+        # control_import_coverage(rootdir)
+        from logilab.devtools.lib.coverage import Coverage
+        cvg = Coverage([rootdir])
+        cvg.erase()
+        cvg.start()
+    if DJANGO_FOUND and options.django:
+        tester = DjangoTester(cvg, options)
+    else:
+        tester = testercls(cvg, options)
+    if explicitfile:
+        cmd, args = tester.testfile, (explicitfile,)
+    elif options.testdir:
+        cmd, args = tester.testonedir, (options.testdir, options.exitfirst)
+    else:
+        cmd, args = tester.testall, (options.exitfirst,)
+    try:
+        try:
+            if options.profile:
+                import hotshot
+                prof = hotshot.Profile(options.profile)
+                prof.runcall(cmd, *args)
+                prof.close()
+                print 'profile data saved in', options.profile
+            else:
+                cmd(*args)
+        except SystemExit:
+            raise
+        except:
+            import traceback
+            traceback.print_exc()
+    finally:
+        if covermode:
+            cvg.stop()
+            cvg.save()
+        tester.show_report()
+        if covermode:
+            print 'coverage information stored, use it with pycoverage -ra'
+        sys.exit(tester.errcode)
+
+class SkipAwareTestProgram(unittest.TestProgram):
+    # XXX: don't try to stay close to unittest.py, use optparse
+    USAGE = """\
+Usage: %(progName)s [options] [test] [...]
+
+Options:
+  -h, --help       Show this message
+  -v, --verbose    Verbose output
+  -i, --pdb        Enable test failure inspection
+  -x, --exitfirst  Exit on first failure
+  -s, --skip       skip test matching this pattern (no regexp for now)
+  -q, --quiet      Minimal output
+  --color          colorize tracebacks
+
+  -m, --match      Run only test whose tag match this pattern
+
+  -P, --profile    FILE: Run the tests using cProfile and saving results
+                   in FILE
+
+Examples:
+  %(progName)s                               - run default set of tests
+  %(progName)s MyTestSuite                   - run suite 'MyTestSuite'
+  %(progName)s MyTestCase.testSomething      - run MyTestCase.testSomething
+  %(progName)s MyTestCase                    - run all 'test*' test methods
+                                               in MyTestCase
+"""
+    def __init__(self, module='__main__', defaultTest=None, batchmode=False,
+                 cvg=None, options=None, outstream=sys.stderr):
+        self.batchmode = batchmode
+        self.cvg = cvg
+        self.options = options
+        self.outstream = outstream
+        super(SkipAwareTestProgram, self).__init__(
+            module=module, defaultTest=defaultTest,
+            testLoader=NonStrictTestLoader())
+
+    def parseArgs(self, argv):
+        self.pdbmode = False
+        self.exitfirst = False
+        self.skipped_patterns = []
+        self.test_pattern = None
+        self.tags_pattern = None
+        self.colorize = False
+        self.profile_name = None
+        import getopt
+        try:
+            options, args = getopt.getopt(argv[1:], 'hHvixrqcp:s:m:P:',
+                                          ['help', 'verbose', 'quiet', 'pdb',
+                                           'exitfirst', 'restart',
+                                           'skip=', 'color', 'match=', 'profile='])
+            for opt, value in options:
+                if opt in ('-h', '-H', '--help'):
+                    self.usageExit()
+                if opt in ('-i', '--pdb'):
+                    self.pdbmode = True
+                if opt in ('-x', '--exitfirst'):
+                    self.exitfirst = True
+                if opt in ('-r', '--restart'):
+                    self.restart = True
+                    self.exitfirst = True
+                if opt in ('-q', '--quiet'):
+                    self.verbosity = 0
+                if opt in ('-v', '--verbose'):
+                    self.verbosity = 2
+                if opt in ('-s', '--skip'):
+                    self.skipped_patterns = [pat.strip() for pat in
+                                             value.split(', ')]
+                if opt == '--color':
+                    self.colorize = True
+                if opt in ('-m', '--match'):
+                    #self.tags_pattern = value
+                    self.options["tag_pattern"] = value
+                if opt in ('-P', '--profile'):
+                    self.profile_name = value
+            self.testLoader.skipped_patterns = self.skipped_patterns
+            if len(args) == 0 and self.defaultTest is None:
+                suitefunc = getattr(self.module, 'suite', None)
+                if isinstance(suitefunc, (types.FunctionType,
+                        types.MethodType)):
+                    self.test = self.module.suite()
+                else:
+                    self.test = self.testLoader.loadTestsFromModule(self.module)
+                return
+            if len(args) > 0:
+                self.test_pattern = args[0]
+                self.testNames = args
+            else:
+                self.testNames = (self.defaultTest, )
+            self.createTests()
+        except getopt.error, msg:
+            self.usageExit(msg)
+
+    def runTests(self):
+        if self.profile_name:
+            import cProfile
+            cProfile.runctx('self._runTests()', globals(), locals(), self.profile_name )
+        else:
+            return self._runTests()
+
+    def _runTests(self):
+        self.testRunner = SkipAwareTextTestRunner(verbosity=self.verbosity,
+                                                  stream=self.outstream,
+                                                  exitfirst=self.exitfirst,
+                                                  pdbmode=self.pdbmode,
+                                                  cvg=self.cvg,
+                                                  test_pattern=self.test_pattern,
+                                                  skipped_patterns=self.skipped_patterns,
+                                                  colorize=self.colorize,
+                                                  batchmode=self.batchmode,
+                                                  options=self.options)
+
+        def removeSucceededTests(obj, succTests):
+            """ Recursive function that removes succTests from
+            a TestSuite or TestCase
+            """
+            if isinstance(obj, unittest.TestSuite):
+                removeSucceededTests(obj._tests, succTests)
+            if isinstance(obj, list):
+                for el in obj[:]:
+                    if isinstance(el, unittest.TestSuite):
+                        removeSucceededTests(el, succTests)
+                    elif isinstance(el, unittest.TestCase):
+                        descr = '.'.join((el.__class__.__module__,
+                                el.__class__.__name__,
+                                el._testMethodName))
+                        if descr in succTests:
+                            obj.remove(el)
+        # take care, self.options may be None
+        if getattr(self.options, 'restart', False):
+            # retrieve succeeded tests from FILE_RESTART
+            try:
+                restartfile = open(FILE_RESTART, 'r')
+                try:
+                    succeededtests = list(elem.rstrip('\n\r') for elem in
+                                          restartfile.readlines())
+                    removeSucceededTests(self.test, succeededtests)
+                finally:
+                    restartfile.close()
+            except Exception, ex:
+                raise Exception("Error while reading succeeded tests into %s: %s"
+                                % (osp.join(os.getcwd(), FILE_RESTART), ex))
+
+        result = self.testRunner.run(self.test)
+        # help garbage collection: we want TestSuite, which hold refs to every
+        # executed TestCase, to be gc'ed
+        del self.test
+        if getattr(result, "debuggers", None) and \
+           getattr(self, "pdbmode", None):
+            start_interactive_mode(result)
+        if not getattr(self, "batchmode", None):
+            sys.exit(not result.wasSuccessful())
+        self.result = result
+
+
+class SkipAwareTextTestRunner(unittest.TextTestRunner):
+
+    def __init__(self, stream=sys.stderr, verbosity=1,
+                 exitfirst=False, pdbmode=False, cvg=None, test_pattern=None,
+                 skipped_patterns=(), colorize=False, batchmode=False,
+                 options=None):
+        super(SkipAwareTextTestRunner, self).__init__(stream=stream,
+                                                      verbosity=verbosity)
+        self.exitfirst = exitfirst
+        self.pdbmode = pdbmode
+        self.cvg = cvg
+        self.test_pattern = test_pattern
+        self.skipped_patterns = skipped_patterns
+        self.colorize = colorize
+        self.batchmode = batchmode
+        self.options = options
+
+    def _this_is_skipped(self, testedname):
+        return any([(pat in testedname) for pat in self.skipped_patterns])
+
+    def _runcondition(self, test, skipgenerator=True):
+        if isinstance(test, testlib.InnerTest):
+            testname = test.name
+        else:
+            if isinstance(test, testlib.TestCase):
+                meth = test._get_test_method()
+                func = meth.im_func
+                testname = '%s.%s' % (meth.im_class.__name__, func.__name__)
+            elif isinstance(test, types.FunctionType):
+                func = test
+                testname = func.__name__
+            elif isinstance(test, types.MethodType):
+                func = test.im_func
+                testname = '%s.%s' % (test.im_class.__name__, func.__name__)
+            else:
+                return True # Not sure when this happens
+            if testlib.is_generator(test) and skipgenerator:
+                return self.does_match_tags(test) # Let inner tests decide at run time
+        if self._this_is_skipped(testname):
+            return False # this was explicitly skipped
+        if self.test_pattern is not None:
+            try:
+                classpattern, testpattern = self.test_pattern.split('.')
+                klass, name = testname.split('.')
+                if classpattern not in klass or testpattern not in name:
+                    return False
+            except ValueError:
+                if self.test_pattern not in testname:
+                    return False
+
+        return self.does_match_tags(test)
+
+    def does_match_tags(self, test):
+        if self.options is not None:
+            tags_pattern = getattr(self.options, 'tags_pattern', None)
+            if tags_pattern is not None:
+                tags = getattr(test, 'tags', testlib.Tags())
+                if tags.inherit and isinstance(test, types.MethodType):
+                    tags = tags | getattr(test.im_class, 'tags', testlib.Tags())
+                return tags.match(tags_pattern)
+        return True # no pattern
+
+    def _makeResult(self):
+        return testlib.SkipAwareTestResult(self.stream, self.descriptions,
+                                   self.verbosity, self.exitfirst,
+                                   self.pdbmode, self.cvg, self.colorize)
+
+    def run(self, test):
+        "Run the given test case or test suite."
+        result = self._makeResult()
+        startTime = time()
+        test(result, runcondition=self._runcondition, options=self.options)
+        stopTime = time()
+        timeTaken = stopTime - startTime
+        result.printErrors()
+        if not self.batchmode:
+            self.stream.writeln(result.separator2)
+            run = result.testsRun
+            self.stream.writeln("Ran %d test%s in %.3fs" %
+                                (run, run != 1 and "s" or "", timeTaken))
+            self.stream.writeln()
+            if not result.wasSuccessful():
+                if self.colorize:
+                    self.stream.write(textutils.colorize_ansi("FAILED", color='red'))
+                else:
+                    self.stream.write("FAILED")
+            else:
+                if self.colorize:
+                    self.stream.write(textutils.colorize_ansi("OK", color='green'))
+                else:
+                    self.stream.write("OK")
+            failed, errored, skipped = map(len, (result.failures,
+                                                 result.errors,
+                                                 result.skipped))
+
+            det_results = []
+            for name, value in (("failures", result.failures),
+                                ("errors",result.errors),
+                                ("skipped", result.skipped)):
+                if value:
+                    det_results.append("%s=%i" % (name, len(value)))
+            if det_results:
+                self.stream.write(" (")
+                self.stream.write(', '.join(det_results))
+                self.stream.write(")")
+            self.stream.writeln("")
+        return result
+
+class NonStrictTestLoader(unittest.TestLoader):
+    """
+    Overrides default testloader to be able to omit classname when
+    specifying tests to run on command line.
+
+    For example, if the file test_foo.py contains ::
+
+        class FooTC(TestCase):
+            def test_foo1(self): # ...
+            def test_foo2(self): # ...
+            def test_bar1(self): # ...
+
+        class BarTC(TestCase):
+            def test_bar2(self): # ...
+
+    'python test_foo.py' will run the 3 tests in FooTC
+    'python test_foo.py FooTC' will run the 3 tests in FooTC
+    'python test_foo.py test_foo' will run test_foo1 and test_foo2
+    'python test_foo.py test_foo1' will run test_foo1
+    'python test_foo.py test_bar' will run FooTC.test_bar1 and BarTC.test_bar2
+    """
+
+    def __init__(self):
+        self.skipped_patterns = ()
+
+    # some magic here to accept empty list by extending
+    # and to provide callable capability
+    def loadTestsFromNames(self, names, module=None):
+        suites = []
+        for name in names:
+            suites.extend(self.loadTestsFromName(name, module))
+        return self.suiteClass(suites)
+
+    def _collect_tests(self, module):
+        tests = {}
+        for obj in vars(module).values():
+            if (issubclass(type(obj), (types.ClassType, type)) and
+                 issubclass(obj, unittest.TestCase)):
+                classname = obj.__name__
+                if classname[0] == '_' or self._this_is_skipped(classname):
+                    continue
+                methodnames = []
+                # obj is a TestCase class
+                for attrname in dir(obj):
+                    if attrname.startswith(self.testMethodPrefix):
+                        attr = getattr(obj, attrname)
+                        if callable(attr):
+                            methodnames.append(attrname)
+                # keep track of class (obj) for convenience
+                tests[classname] = (obj, methodnames)
+        return tests
+
+    def loadTestsFromSuite(self, module, suitename):
+        try:
+            suite = getattr(module, suitename)()
+        except AttributeError:
+            return []
+        assert hasattr(suite, '_tests'), \
+               "%s.%s is not a valid TestSuite" % (module.__name__, suitename)
+        # python2.3 does not implement __iter__ on suites, we need to return
+        # _tests explicitly
+        return suite._tests
+
+    def loadTestsFromName(self, name, module=None):
+        parts = name.split('.')
+        if module is None or len(parts) > 2:
+            # let the base class do its job here
+            return [super(NonStrictTestLoader, self).loadTestsFromName(name)]
+        tests = self._collect_tests(module)
+        collected = []
+        if len(parts) == 1:
+            pattern = parts[0]
+            if callable(getattr(module, pattern, None)
+                    )  and pattern not in tests:
+                # consider it as a suite
+                return self.loadTestsFromSuite(module, pattern)
+            if pattern in tests:
+                # case python unittest_foo.py MyTestTC
+                klass, methodnames = tests[pattern]
+                for methodname in methodnames:
+                    collected = [klass(methodname)
+                        for methodname in methodnames]
+            else:
+                # case python unittest_foo.py something
+                for klass, methodnames in tests.values():
+                    # skip methodname if matched by skipped_patterns
+                    for skip_pattern in self.skipped_patterns:
+                        methodnames = [methodname
+                                      for methodname in methodnames
+                                      if skip_pattern not in methodname]
+                    collected += [klass(methodname)
+                                  for methodname in methodnames
+                                  if pattern in methodname]
+        elif len(parts) == 2:
+            # case "MyClass.test_1"
+            classname, pattern = parts
+            klass, methodnames = tests.get(classname, (None, []))
+            for methodname in methodnames:
+                collected = [klass(methodname) for methodname in methodnames
+                            if pattern in methodname]
+        return collected
+
+    def _this_is_skipped(self, testedname):
+        return any([(pat in testedname) for pat in self.skipped_patterns])
+
+    def getTestCaseNames(self, testCaseClass):
+        """Return a sorted sequence of method names found within testCaseClass
+        """
+        is_skipped = self._this_is_skipped
+        classname = testCaseClass.__name__
+        if classname[0] == '_' or is_skipped(classname):
+            return []
+        testnames = super(NonStrictTestLoader, self).getTestCaseNames(
+                testCaseClass)
+        return [testname for testname in testnames if not is_skipped(testname)]
+
+def _ts_run(self, result, runcondition=None, options=None):
+    self._wrapped_run(result,runcondition=runcondition, options=options)
+    self._tearDownPreviousClass(None, result)
+    self._handleModuleTearDown(result)
+    return result
+
+def _ts_wrapped_run(self, result, debug=False, runcondition=None, options=None):
+    for test in self:
+        if result.shouldStop:
+            break
+        if unittest_suite._isnotsuite(test):
+            self._tearDownPreviousClass(test, result)
+            self._handleModuleFixture(test, result)
+            self._handleClassSetUp(test, result)
+            result._previousTestClass = test.__class__
+            if (getattr(test.__class__, '_classSetupFailed', False) or 
+                getattr(result, '_moduleSetUpFailed', False)):
+                continue
+
+        if hasattr(test, '_wrapped_run'):
+            try:
+                test._wrapped_run(result, debug, runcondition=runcondition, options=options)
+            except TypeError:
+                test._wrapped_run(result, debug)
+        elif not debug:
+            try:
+                test(result, runcondition, options)
+            except TypeError:
+                test(result)
+        else:
+            test.debug()
+
+
+def enable_dbc(*args):
+    """
+    Without arguments, return True if contracts can be enabled and should be
+    enabled (see option -d), return False otherwise.
+
+    With arguments, return False if contracts can't or shouldn't be enabled,
+    otherwise weave ContractAspect with items passed as arguments.
+    """
+    if not ENABLE_DBC:
+        return False
+    try:
+        from logilab.aspects.weaver import weaver
+        from logilab.aspects.lib.contracts import ContractAspect
+    except ImportError:
+        sys.stderr.write(
+            'Warning: logilab.aspects is not available. Contracts disabled.')
+        return False
+    for arg in args:
+        weaver.weave_module(arg, ContractAspect)
+    return True
+
+
+# monkeypatch unittest and doctest (ouch !)
+unittest._TextTestResult = testlib.SkipAwareTestResult
+unittest.TextTestRunner = SkipAwareTextTestRunner
+unittest.TestLoader = NonStrictTestLoader
+unittest.TestProgram = SkipAwareTestProgram
+
+if sys.version_info >= (2, 4):
+    doctest.DocTestCase.__bases__ = (testlib.TestCase,)
+    # XXX check python2.6 compatibility
+    #doctest.DocTestCase._cleanups = []
+    #doctest.DocTestCase._out = []
+else:
+    unittest.FunctionTestCase.__bases__ = (testlib.TestCase,)
+unittest.TestSuite.run = _ts_run
+unittest.TestSuite._wrapped_run = _ts_wrapped_run

pylibs/logilab/common/registry.py

@@ -0,0 +1,973 @@
+# copyright 2003-2012 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+#
+# This file is part of Logilab-common.
+#
+# Logilab-common is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Lesser General Public License as published by the
+# Free Software Foundation, either version 2.1 of the License, or (at your
+# option) any later version.
+#
+# Logilab-common is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with Logilab-common.  If not, see <http://www.gnu.org/licenses/>.
+"""This module provides bases for predicates dispatching (the pattern in use
+here is similar to what's refered as multi-dispatch or predicate-dispatch in the
+literature, though a bit different since the idea is to select across different
+implementation 'e.g. classes), not to dispatch a message to a function or
+method. It contains the following classes:
+
+* :class:`RegistryStore`, the top level object which loads implementation
+  objects and stores them into registries. You'll usually use it to access
+  registries and their contained objects;
+
+* :class:`Registry`, the base class which contains objects semantically grouped
+  (for instance, sharing a same API, hence the 'implementation' name). You'll
+  use it to select the proper implementation according to a context. Notice you
+  may use registries on their own without using the store.
+
+.. Note::
+
+  implementation objects are usually designed to be accessed through the
+  registry and not by direct instantiation, besides to use it as base classe.
+
+The selection procedure is delegated to a selector, which is responsible for
+scoring the object according to some context. At the end of the selection, if an
+implementation has been found, an instance of this class is returned. A selector
+is built from one or more predicates combined together using AND, OR, NOT
+operators (actually `&`, `|` and `~`). You'll thus find some base classes to
+build predicates:
+
+* :class:`Predicate`, the abstract base predicate class
+
+* :class:`AndPredicate`, :class:`OrPredicate`, :class:`NotPredicate`, which you
+  shouldn't have to use directly. You'll use `&`, `|` and '~' operators between
+  predicates directly
+
+* :func:`objectify_predicate`
+
+You'll eventually find one concrete predicate: :class:`yes`
+
+.. autoclass:: RegistryStore
+.. autoclass:: Registry
+
+Predicates
+----------
+.. autoclass:: Predicate
+.. autofunc:: objectify_predicate
+.. autoclass:: yes
+
+Debugging
+---------
+.. autoclass:: traced_selection
+
+Exceptions
+----------
+.. autoclass:: RegistryException
+.. autoclass:: RegistryNotFound
+.. autoclass:: ObjectNotFound
+.. autoclass:: NoSelectableObject
+"""
+
+__docformat__ = "restructuredtext en"
+
+import sys
+import types
+import weakref
+from os import listdir, stat
+from os.path import join, isdir, exists
+from logging import getLogger
+
+from logilab.common.logging_ext import set_log_methods
+
+
+class RegistryException(Exception):
+    """Base class for registry exception."""
+
+class RegistryNotFound(RegistryException):
+    """Raised when an unknown registry is requested.
+
+    This is usually a programming/typo error.
+    """
+
+class ObjectNotFound(RegistryException):
+    """Raised when an unregistered object is requested.
+
+    This may be a programming/typo or a misconfiguration error.
+    """
+
+class NoSelectableObject(RegistryException):
+    """Raised when no object is selectable for a given context."""
+    def __init__(self, args, kwargs, objects):
+        self.args = args
+        self.kwargs = kwargs
+        self.objects = objects
+
+    def __str__(self):
+        return ('args: %s, kwargs: %s\ncandidates: %s'
+                % (self.args, self.kwargs.keys(), self.objects))
+
+
+def _toload_info(path, extrapath, _toload=None):
+    """Return a dictionary of <modname>: <modpath> and an ordered list of
+    (file, module name) to load
+    """
+    from logilab.common.modutils import modpath_from_file
+    if _toload is None:
+        assert isinstance(path, list)
+        _toload = {}, []
+    for fileordir in path:
+        if isdir(fileordir) and exists(join(fileordir, '__init__.py')):
+            subfiles = [join(fileordir, fname) for fname in listdir(fileordir)]
+            _toload_info(subfiles, extrapath, _toload)
+        elif fileordir[-3:] == '.py':
+            modpath = modpath_from_file(fileordir, extrapath)
+            # omit '__init__' from package's name to avoid loading that module
+            # once for each name when it is imported by some other object
+            # module. This supposes import in modules are done as::
+            #
+            #   from package import something
+            #
+            # not::
+            #
+            #   from package.__init__ import something
+            #
+            # which seems quite correct.
+            if modpath[-1] == '__init__':
+                modpath.pop()
+            modname = '.'.join(modpath)
+            _toload[0][modname] = fileordir
+            _toload[1].append((fileordir, modname))
+    return _toload
+
+
+def classid(cls):
+    """returns a unique identifier for an object class"""
+    return '%s.%s' % (cls.__module__, cls.__name__)
+
+def class_registries(cls, registryname):
+    """return a tuple of registry names (see __registries__)"""
+    if registryname:
+        return (registryname,)
+    return cls.__registries__
+
+
+class Registry(dict):
+    """The registry store a set of implementations associated to identifier:
+
+    * to each identifier are associated a list of implementations
+
+    * to select an implementation of a given identifier, you should use one of the
+      :meth:`select` or :meth:`select_or_none` method
+
+    * to select a list of implementations for a context, you should use the
+      :meth:`possible_objects` method
+
+    * dictionary like access to an identifier will return the bare list of
+      implementations for this identifier.
+
+    To be usable in a registry, the only requirement is to have a `__select__`
+    attribute.
+
+    At the end of the registration process, the :meth:`__registered__`
+    method is called on each registered object which have them, given the
+    registry in which it's registered as argument.
+
+    Registration methods:
+
+    .. automethod: register
+    .. automethod: unregister
+
+    Selection methods:
+
+    .. automethod: select
+    .. automethod: select_or_none
+    .. automethod: possible_objects
+    .. automethod: object_by_id
+    """
+    def __init__(self, debugmode):
+        super(Registry, self).__init__()
+        self.debugmode = debugmode
+
+    def __getitem__(self, name):
+        """return the registry (list of implementation objects) associated to
+        this name
+        """
+        try:
+            return super(Registry, self).__getitem__(name)
+        except KeyError:
+            raise ObjectNotFound(name), None, sys.exc_info()[-1]
+
+    def initialization_completed(self):
+        """call method __registered__() on registered objects when the callback
+        is defined"""
+        for objects in self.itervalues():
+            for objectcls in objects:
+                registered = getattr(objectcls, '__registered__', None)
+                if registered:
+                    registered(self)
+        if self.debugmode:
+            wrap_predicates(_lltrace)
+
+    def register(self, obj, oid=None, clear=False):
+        """base method to add an object in the registry"""
+        assert not '__abstract__' in obj.__dict__
+        assert obj.__select__
+        oid = oid or obj.__regid__
+        assert oid
+        if clear:
+            objects = self[oid] =  []
+        else:
+            objects = self.setdefault(oid, [])
+        assert not obj in objects, \
+               'object %s is already registered' % obj
+        objects.append(obj)
+
+    def register_and_replace(self, obj, replaced):
+        """remove <replaced> and register <obj>"""
+        # XXXFIXME this is a duplication of unregister()
+        # remove register_and_replace in favor of unregister + register
+        # or simplify by calling unregister then register here
+        if not isinstance(replaced, basestring):
+            replaced = classid(replaced)
+        # prevent from misspelling
+        assert obj is not replaced, 'replacing an object by itself: %s' % obj
+        registered_objs = self.get(obj.__regid__, ())
+        for index, registered in enumerate(registered_objs):
+            if classid(registered) == replaced:
+                del registered_objs[index]
+                break
+        else:
+            self.warning('trying to replace %s that is not registered with %s',
+                         replaced, obj)
+        self.register(obj)
+
+    def unregister(self, obj):
+        """remove object <obj> from this registry"""
+        clsid = classid(obj)
+        oid = obj.__regid__
+        for registered in self.get(oid, ()):
+            # use classid() to compare classes because vreg will probably
+            # have its own version of the class, loaded through execfile
+            if classid(registered) == clsid:
+                self[oid].remove(registered)
+                break
+        else:
+            self.warning('can\'t remove %s, no id %s in the registry',
+                         clsid, oid)
+
+    def all_objects(self):
+        """return a list containing all objects in this registry.
+        """
+        result = []
+        for objs in self.values():
+            result += objs
+        return result
+
+    # dynamic selection methods ################################################
+
+    def object_by_id(self, oid, *args, **kwargs):
+        """return object with the `oid` identifier. Only one object is expected
+        to be found.
+
+        raise :exc:`ObjectNotFound` if not object with id <oid> in <registry>
+
+        raise :exc:`AssertionError` if there is more than one object there
+        """
+        objects = self[oid]
+        assert len(objects) == 1, objects
+        return objects[0](*args, **kwargs)
+
+    def select(self, __oid, *args, **kwargs):
+        """return the most specific object among those with the given oid
+        according to the given context.
+
+        raise :exc:`ObjectNotFound` if not object with id <oid> in <registry>
+
+        raise :exc:`NoSelectableObject` if not object apply
+        """
+        obj =  self._select_best(self[__oid], *args, **kwargs)
+        if obj is None:
+            raise NoSelectableObject(args, kwargs, self[__oid] )
+        return obj
+
+    def select_or_none(self, __oid, *args, **kwargs):
+        """return the most specific object among those with the given oid
+        according to the given context, or None if no object applies.
+        """
+        try:
+            return self.select(__oid, *args, **kwargs)
+        except (NoSelectableObject, ObjectNotFound):
+            return None
+
+    def possible_objects(self, *args, **kwargs):
+        """return an iterator on possible objects in this registry for the given
+        context
+        """
+        for objects in self.itervalues():
+            obj = self._select_best(objects,  *args, **kwargs)
+            if obj is None:
+                continue
+            yield obj
+
+    def _select_best(self, objects, *args, **kwargs):
+        """return an instance of the most specific object according
+        to parameters
+
+        return None if not object apply (don't raise `NoSelectableObject` since
+        it's costly when searching objects using `possible_objects`
+        (e.g. searching for hooks).
+        """
+        score, winners = 0, None
+        for obj in objects:
+            objectscore = obj.__select__(obj, *args, **kwargs)
+            if objectscore > score:
+                score, winners = objectscore, [obj]
+            elif objectscore > 0 and objectscore == score:
+                winners.append(obj)
+        if winners is None:
+            return None
+        if len(winners) > 1:
+            # log in production environement / test, error while debugging
+            msg = 'select ambiguity: %s\n(args: %s, kwargs: %s)'
+            if self.debugmode:
+                # raise bare exception in debug mode
+                raise Exception(msg % (winners, args, kwargs.keys()))
+            self.error(msg, winners, args, kwargs.keys())
+        # return the result of calling the object
+        return winners[0](*args, **kwargs)
+
+    # these are overridden by set_log_methods below
+    # only defining here to prevent pylint from complaining
+    info = warning = error = critical = exception = debug = lambda msg, *a, **kw: None
+
+
+class RegistryStore(dict):
+    """This class is responsible for loading implementations and storing them
+    in their registry which are created on the fly as needed.
+
+    It handles dynamic registration of objects and provides a convenient api to
+    access them. To be recognized as an object that should be stored into one of
+    the store's registry (:class:`Registry`), an object (usually a class) has
+    the following attributes, used control how they interact with the registry:
+
+    :attr:`__registry__` or `__registries__`
+      name of the registry for this object (string like 'views', 'templates'...)
+      or list of registry names if you want your object to be added to multiple
+      registries
+
+    :attr:`__regid__`
+      implementation's identifier in the registry (string like 'main',
+      'primary', 'folder_box')
+
+    :attr:`__select__`
+      the implementation's selector
+
+    Moreover, the :attr:`__abstract__` attribute may be set to `True` to
+    indicate that a class is abstract and should not be registered (inherited
+    attributes not considered).
+
+    .. Note::
+
+      When using the store to load objects dynamically, you *always* have
+      to use **super()** to get the methods and attributes of the
+      superclasses, and not use the class identifier. Else, you'll get into
+      trouble when reloading comes into the place.
+
+      For example, instead of writing::
+
+          class Thing(Parent):
+              __regid__ = 'athing'
+              __select__ = yes()
+              def f(self, arg1):
+                  Parent.f(self, arg1)
+
+      You must write::
+
+          class Thing(Parent):
+              __regid__ = 'athing'
+              __select__ = yes()
+              def f(self, arg1):
+                  super(Parent, self).f(arg1)
+
+    Controlling objects registration
+    --------------------------------
+
+    Dynamic loading is triggered by calling the :meth:`register_objects` method,
+    given a list of directory to inspect for python modules.
+
+    .. automethod: register_objects
+
+    For each module, by default, all compatible objects are registered
+    automatically, though if some objects have to replace other objects, or have
+    to be included only if some condition is met, you'll have to define a
+    `registration_callback(vreg)` function in your module and explicitly
+    register **all objects** in this module, using the api defined below.
+
+
+    .. automethod:: RegistryStore.register_all
+    .. automethod:: RegistryStore.register_and_replace
+    .. automethod:: RegistryStore.register
+    .. automethod:: RegistryStore.unregister
+
+    .. Note::
+        Once the function `registration_callback(vreg)` is implemented in a
+        module, all the objects from this module have to be explicitly
+        registered as it disables the automatic objects registration.
+
+
+    Examples:
+
+    .. sourcecode:: python
+
+       # cubicweb/web/views/basecomponents.py
+       def registration_callback(store):
+          # register everything in the module except SeeAlsoComponent
+          store.register_all(globals().values(), __name__, (SeeAlsoVComponent,))
+          # conditionally register SeeAlsoVComponent
+          if 'see_also' in store.schema:
+              store.register(SeeAlsoVComponent)
+
+    In this example, we register all application object classes defined in the module
+    except `SeeAlsoVComponent`. This class is then registered only if the 'see_also'
+    relation type is defined in the instance'schema.
+
+    .. sourcecode:: python
+
+       # goa/appobjects/sessions.py
+       def registration_callback(store):
+          store.register(SessionsCleaner)
+          # replace AuthenticationManager by GAEAuthenticationManager
+          store.register_and_replace(GAEAuthenticationManager, AuthenticationManager)
+          # replace PersistentSessionManager by GAEPersistentSessionManager
+          store.register_and_replace(GAEPersistentSessionManager, PersistentSessionManager)
+
+    In this example, we explicitly register classes one by one:
+
+    * the `SessionCleaner` class
+    * the `GAEAuthenticationManager` to replace the `AuthenticationManager`
+    * the `GAEPersistentSessionManager` to replace the `PersistentSessionManager`
+
+    If at some point we register a new appobject class in this module, it won't be
+    registered at all without modification to the `registration_callback`
+    implementation. The previous example will register it though, thanks to the call
+    to the `register_all` method.
+
+    Controlling registry instantation
+    ---------------------------------
+    The `REGISTRY_FACTORY` class dictionary allows to specify which class should
+    be instantiated for a given registry name. The class associated to `None` in
+    it will be the class used when there is no specific class for a name.
+    """
+
+    def __init__(self, debugmode=False):
+        super(RegistryStore, self).__init__()
+        self.debugmode = debugmode
+
+    def reset(self):
+        """clear all registries managed by this store"""
+        # don't use self.clear, we want to keep existing subdictionaries
+        for subdict in self.itervalues():
+            subdict.clear()
+        self._lastmodifs = {}
+
+    def __getitem__(self, name):
+        """return the registry (dictionary of class objects) associated to
+        this name
+        """
+        try:
+            return super(RegistryStore, self).__getitem__(name)
+        except KeyError:
+            raise RegistryNotFound(name), None, sys.exc_info()[-1]
+
+    # methods for explicit (un)registration ###################################
+
+    # default class, when no specific class set
+    REGISTRY_FACTORY = {None: Registry}
+
+    def registry_class(self, regid):
+        """return existing registry named regid or use factory to create one and
+        return it"""
+        try:
+            return self.REGISTRY_FACTORY[regid]
+        except KeyError:
+            return self.REGISTRY_FACTORY[None]
+
+    def setdefault(self, regid):
+        try:
+            return self[regid]
+        except KeyError:
+            self[regid] = self.registry_class(regid)(self.debugmode)
+            return self[regid]
+
+    def register_all(self, objects, modname, butclasses=()):
+        """register all `objects` given. Objects which are not from the module
+        `modname` or which are in `butclasses` won't be registered.
+
+        Typical usage is:
+
+        .. sourcecode:: python
+
+            store.register_all(globals().values(), __name__, (ClassIWantToRegisterExplicitly,))
+
+        So you get partially automatic registration, keeping manual registration
+        for some object (to use
+        :meth:`~logilab.common.registry.RegistryStore.register_and_replace`
+        for instance)
+        """
+        for obj in objects:
+            try:
+                if obj.__module__ != modname or obj in butclasses:
+                    continue
+                oid = obj.__regid__
+            except AttributeError:
+                continue
+            if oid and not obj.__dict__.get('__abstract__'):
+                self.register(obj, oid=oid)
+
+    def register(self, obj, registryname=None, oid=None, clear=False):
+        """register `obj` implementation into `registryname` or
+        `obj.__registry__` if not specified, with identifier `oid` or
+        `obj.__regid__` if not specified.
+
+        If `clear` is true, all objects with the same identifier will be
+        previously unregistered.
+        """
+        assert not obj.__dict__.get('__abstract__')
+        try:
+            vname = obj.__name__
+        except AttributeError:
+            # XXX may occurs?
+            vname = obj.__class__.__name__
+        for registryname in class_registries(obj, registryname):
+            registry = self.setdefault(registryname)
+            registry.register(obj, oid=oid, clear=clear)
+            self.debug('register %s in %s[\'%s\']',
+                       vname, registryname, oid or obj.__regid__)
+        self._loadedmods.setdefault(obj.__module__, {})[classid(obj)] = obj
+
+    def unregister(self, obj, registryname=None):
+        """unregister `obj` implementation object from the registry
+        `registryname` or `obj.__registry__` if not specified.
+        """
+        for registryname in class_registries(obj, registryname):
+            self[registryname].unregister(obj)
+
+    def register_and_replace(self, obj, replaced, registryname=None):
+        """register `obj` implementation object into `registryname` or
+        `obj.__registry__` if not specified. If found, the `replaced` object
+        will be unregistered first (else a warning will be issued as it's
+        generally unexpected).
+        """
+        for registryname in class_registries(obj, registryname):
+            self[registryname].register_and_replace(obj, replaced)
+
+    # initialization methods ###################################################
+
+    def init_registration(self, path, extrapath=None):
+        """reset registry and walk down path to return list of (path, name)
+        file modules to be loaded"""
+        # XXX make this private by renaming it to _init_registration ?
+        self.reset()
+        # compute list of all modules that have to be loaded
+        self._toloadmods, filemods = _toload_info(path, extrapath)
+        # XXX is _loadedmods still necessary ? It seems like it's useful
+        #     to avoid loading same module twice, especially with the
+        #     _load_ancestors_then_object logic but this needs to be checked
+        self._loadedmods = {}
+        return filemods
+
+    def register_objects(self, path, extrapath=None):
+        """register all objects found walking down <path>"""
+        # load views from each directory in the instance's path
+        # XXX inline init_registration ?
+        filemods = self.init_registration(path, extrapath)
+        for filepath, modname in filemods:
+            self.load_file(filepath, modname)
+        self.initialization_completed()
+
+    def initialization_completed(self):
+        """call initialization_completed() on all known registries"""
+        for reg in self.itervalues():
+            reg.initialization_completed()
+
+    def _mdate(self, filepath):
+        try:
+            return stat(filepath)[-2]
+        except OSError:
+            # this typically happens on emacs backup files (.#foo.py)
+            self.warning('Unable to load %s. It is likely to be a backup file',
+                         filepath)
+            return None
+
+    def is_reload_needed(self, path):
+        """return True if something module changed and the registry should be
+        reloaded
+        """
+        lastmodifs = self._lastmodifs
+        for fileordir in path:
+            if isdir(fileordir) and exists(join(fileordir, '__init__.py')):
+                if self.is_reload_needed([join(fileordir, fname)
+                                          for fname in listdir(fileordir)]):
+                    return True
+            elif fileordir[-3:] == '.py':
+                mdate = self._mdate(fileordir)
+                if mdate is None:
+                    continue # backup file, see _mdate implementation
+                elif "flymake" in fileordir:
+                    # flymake + pylint in use, don't consider these they will corrupt the registry
+                    continue
+                if fileordir not in lastmodifs or lastmodifs[fileordir] < mdate:
+                    self.info('File %s changed since last visit', fileordir)
+                    return True
+        return False
+
+    def load_file(self, filepath, modname):
+        """load app objects from a python file"""
+        from logilab.common.modutils import load_module_from_name
+        if modname in self._loadedmods:
+            return
+        self._loadedmods[modname] = {}
+        mdate = self._mdate(filepath)
+        if mdate is None:
+            return # backup file, see _mdate implementation
+        elif "flymake" in filepath:
+            # flymake + pylint in use, don't consider these they will corrupt the registry
+            return
+        # set update time before module loading, else we get some reloading
+        # weirdness in case of syntax error or other error while importing the
+        # module
+        self._lastmodifs[filepath] = mdate
+        # load the module
+        module = load_module_from_name(modname)
+        self.load_module(module)
+
+    def load_module(self, module):
+        """load objects from a module using registration_callback() when it exists
+        """
+        self.info('loading %s from %s', module.__name__, module.__file__)
+        if hasattr(module, 'registration_callback'):
+            module.registration_callback(self)
+        else:
+            for objname, obj in vars(module).items():
+                if objname.startswith('_'):
+                    continue
+                self._load_ancestors_then_object(module.__name__, obj)
+
+    def _load_ancestors_then_object(self, modname, objectcls):
+        """handle automatic object class registration:
+
+        - first ensure parent classes are already registered
+
+        - class with __abstract__ == True in their local dictionary or
+          with a name starting with an underscore are not registered
+
+        - object class needs to have __registry__ and __regid__ attributes
+          set to a non empty string to be registered.
+        """
+        # imported classes
+        objmodname = getattr(objectcls, '__module__', None)
+        if objmodname != modname:
+            if objmodname in self._toloadmods:
+                self.load_file(self._toloadmods[objmodname], objmodname)
+            return
+        # skip non registerable object
+        try:
+            if not (getattr(objectcls, '__regid__', None)
+                    and getattr(objectcls, '__select__', None)):
+                return
+        except TypeError:
+            return
+        clsid = classid(objectcls)
+        if clsid in self._loadedmods[modname]:
+            return
+        self._loadedmods[modname][clsid] = objectcls
+        for parent in objectcls.__bases__:
+            self._load_ancestors_then_object(modname, parent)
+        if (objectcls.__dict__.get('__abstract__')
+            or objectcls.__name__[0] == '_'
+            or not objectcls.__registries__
+            or not objectcls.__regid__):
+            return
+        try:
+            self.register(objectcls)
+        except Exception, ex:
+            if self.debugmode:
+                raise
+            self.exception('object %s registration failed: %s',
+                           objectcls, ex)
+
+    # these are overridden by set_log_methods below
+    # only defining here to prevent pylint from complaining
+    info = warning = error = critical = exception = debug = lambda msg, *a, **kw: None
+
+
+# init logging
+set_log_methods(RegistryStore, getLogger('registry.store'))
+set_log_methods(Registry, getLogger('registry'))
+
+
+# helpers for debugging selectors
+TRACED_OIDS = None
+
+def _trace_selector(cls, selector, args, ret):
+    vobj = args[0]
+    if TRACED_OIDS == 'all' or vobj.__regid__ in TRACED_OIDS:
+        print '%s -> %s for %s(%s)' % (cls, ret, vobj, vobj.__regid__)
+
+def _lltrace(selector):
+    """use this decorator on your predicates so they become traceable with
+    :class:`traced_selection`
+    """
+    def traced(cls, *args, **kwargs):
+        ret = selector(cls, *args, **kwargs)
+        if TRACED_OIDS is not None:
+            _trace_selector(cls, selector, args, ret)
+        return ret
+    traced.__name__ = selector.__name__
+    traced.__doc__ = selector.__doc__
+    return traced
+
+class traced_selection(object): # pylint: disable=C0103
+    """
+    Typical usage is :
+
+    .. sourcecode:: python
+
+        >>> from logilab.common.registry import traced_selection
+        >>> with traced_selection():
+        ...     # some code in which you want to debug selectors
+        ...     # for all objects
+
+    Don't forget the 'from __future__ import with_statement' at the module top-level
+    if you're using python prior to 2.6.
+
+    This will yield lines like this in the logs::
+
+        selector one_line_rset returned 0 for <class 'cubicweb.web.views.basecomponents.WFHistoryVComponent'>
+
+    You can also give to :class:`traced_selection` the identifiers of objects on
+    which you want to debug selection ('oid1' and 'oid2' in the example above).
+
+    .. sourcecode:: python
+
+        >>> with traced_selection( ('regid1', 'regid2') ):
+        ...     # some code in which you want to debug selectors
+        ...     # for objects with __regid__ 'regid1' and 'regid2'
+
+    A potentially useful point to set up such a tracing function is
+    the `logilab.common.registry.Registry.select` method body.
+    """
+
+    def __init__(self, traced='all'):
+        self.traced = traced
+
+    def __enter__(self):
+        global TRACED_OIDS
+        TRACED_OIDS = self.traced
+
+    def __exit__(self, exctype, exc, traceback):
+        global TRACED_OIDS
+        TRACED_OIDS = None
+        return traceback is None
+
+# selector base classes and operations ########################################
+
+def objectify_predicate(selector_func):
+    """Most of the time, a simple score function is enough to build a selector.
+    The :func:`objectify_predicate` decorator turn it into a proper selector
+    class::
+
+        @objectify_predicate
+        def one(cls, req, rset=None, **kwargs):
+            return 1
+
+        class MyView(View):
+            __select__ = View.__select__ & one()
+
+    """
+    return type(selector_func.__name__, (Predicate,),
+                {'__doc__': selector_func.__doc__,
+                 '__call__': lambda self, *a, **kw: selector_func(*a, **kw)})
+
+
+_PREDICATES = {}
+
+def wrap_predicates(decorator):
+    for predicate in _PREDICATES.itervalues():
+        if not '_decorators' in predicate.__dict__:
+            predicate._decorators = set()
+        if decorator in predicate._decorators:
+            continue
+        predicate._decorators.add(decorator)
+        predicate.__call__ = decorator(predicate.__call__)
+
+class PredicateMetaClass(type):
+    def __new__(cls, *args, **kwargs):
+        # use __new__ so subclasses doesn't have to call Predicate.__init__
+        inst = type.__new__(cls, *args, **kwargs)
+        proxy = weakref.proxy(inst, lambda p: _PREDICATES.pop(id(p)))
+        _PREDICATES[id(proxy)] = proxy
+        return inst
+
+class Predicate(object):
+    """base class for selector classes providing implementation
+    for operators ``&``, ``|`` and  ``~``
+
+    This class is only here to give access to binary operators, the selector
+    logic itself should be implemented in the :meth:`__call__` method. Notice it
+    should usually accept any arbitrary arguments (the context), though that may
+    vary depending on your usage of the registry.
+
+    a selector is called to help choosing the correct object for a
+    particular context by returning a score (`int`) telling how well
+    the implementation given as first argument fit to the given context.
+
+    0 score means that the class doesn't apply.
+    """
+    __metaclass__ = PredicateMetaClass
+
+    @property
+    def func_name(self):
+        # backward compatibility
+        return self.__class__.__name__
+
+    def search_selector(self, selector):
+        """search for the given selector, selector instance or tuple of
+        selectors in the selectors tree. Return None if not found.
+        """
+        if self is selector:
+            return self
+        if (isinstance(selector, type) or isinstance(selector, tuple)) and \
+               isinstance(self, selector):
+            return self
+        return None
+
+    def __str__(self):
+        return self.__class__.__name__
+
+    def __and__(self, other):
+        return AndPredicate(self, other)
+    def __rand__(self, other):
+        return AndPredicate(other, self)
+    def __iand__(self, other):
+        return AndPredicate(self, other)
+    def __or__(self, other):
+        return OrPredicate(self, other)
+    def __ror__(self, other):
+        return OrPredicate(other, self)
+    def __ior__(self, other):
+        return OrPredicate(self, other)
+
+    def __invert__(self):
+        return NotPredicate(self)
+
+    # XXX (function | function) or (function & function) not managed yet
+
+    def __call__(self, cls, *args, **kwargs):
+        return NotImplementedError("selector %s must implement its logic "
+                                   "in its __call__ method" % self.__class__)
+
+    def __repr__(self):
+        return u'<Predicate %s at %x>' % (self.__class__.__name__, id(self))
+
+
+class MultiPredicate(Predicate):
+    """base class for compound selector classes"""
+
+    def __init__(self, *selectors):
+        self.selectors = self.merge_selectors(selectors)
+
+    def __str__(self):
+        return '%s(%s)' % (self.__class__.__name__,
+                           ','.join(str(s) for s in self.selectors))
+
+    @classmethod
+    def merge_selectors(cls, selectors):
+        """deal with selector instanciation when necessary and merge
+        multi-selectors if possible:
+
+        AndPredicate(AndPredicate(sel1, sel2), AndPredicate(sel3, sel4))
+        ==> AndPredicate(sel1, sel2, sel3, sel4)
+        """
+        merged_selectors = []
+        for selector in selectors:
+            # XXX do we really want magic-transformations below?
+            # if so, wanna warn about them?
+            if isinstance(selector, types.FunctionType):
+                selector = objectify_predicate(selector)()
+            if isinstance(selector, type) and issubclass(selector, Predicate):
+                selector = selector()
+            assert isinstance(selector, Predicate), selector
+            if isinstance(selector, cls):
+                merged_selectors += selector.selectors
+            else:
+                merged_selectors.append(selector)
+        return merged_selectors
+
+    def search_selector(self, selector):
+        """search for the given selector or selector instance (or tuple of
+        selectors) in the selectors tree. Return None if not found
+        """
+        for childselector in self.selectors:
+            if childselector is selector:
+                return childselector
+            found = childselector.search_selector(selector)
+            if found is not None:
+                return found
+        # if not found in children, maybe we are looking for self?
+        return super(MultiPredicate, self).search_selector(selector)
+
+
+class AndPredicate(MultiPredicate):
+    """and-chained selectors"""
+    def __call__(self, cls, *args, **kwargs):
+        score = 0
+        for selector in self.selectors:
+            partscore = selector(cls, *args, **kwargs)
+            if not partscore:
+                return 0
+            score += partscore
+        return score
+
+
+class OrPredicate(MultiPredicate):
+    """or-chained selectors"""
+    def __call__(self, cls, *args, **kwargs):
+        for selector in self.selectors:
+            partscore = selector(cls, *args, **kwargs)
+            if partscore:
+                return partscore
+        return 0
+
+class NotPredicate(Predicate):
+    """negation selector"""
+    def __init__(self, selector):
+        self.selector = selector
+
+    def __call__(self, cls, *args, **kwargs):
+        score = self.selector(cls, *args, **kwargs)
+        return int(not score)
+
+    def __str__(self):
+        return 'NOT(%s)' % self.selector
+
+
+class yes(Predicate): # pylint: disable=C0103
+    """Return the score given as parameter, with a default score of 0.5 so any
+    other selector take precedence.
+
+    Usually used for objects which can be selected whatever the context, or
+    also sometimes to add arbitrary points to a score.
+
+    Take care, `yes(0)` could be named 'no'...
+    """
+    def __init__(self, score=0.5):
+        self.score = score
+
+    def __call__(self, *args, **kwargs):
+        return self.score

pylibs/logilab/common/shellutils.py

@@ -0,0 +1,456 @@
+# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+#
+# This file is part of logilab-common.
+#
+# logilab-common is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation, either version 2.1 of the License, or (at your option) any
+# later version.
+#
+# logilab-common is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with logilab-common.  If not, see <http://www.gnu.org/licenses/>.
+"""shell/term utilities, useful to write some python scripts instead of shell
+scripts.
+"""
+__docformat__ = "restructuredtext en"
+
+import os
+import glob
+import shutil
+import stat
+import sys
+import tempfile
+import time
+import fnmatch
+import errno
+import string
+import random
+from os.path import exists, isdir, islink, basename, join
+
+from logilab.common import STD_BLACKLIST, _handle_blacklist
+from logilab.common.compat import raw_input
+from logilab.common.compat import str_to_bytes
+
+try:
+    from logilab.common.proc import ProcInfo, NoSuchProcess
+except ImportError:
+    # windows platform
+    class NoSuchProcess(Exception): pass
+
+    def ProcInfo(pid):
+        raise NoSuchProcess()
+
+
+class tempdir(object):
+
+    def __enter__(self):
+        self.path = tempfile.mkdtemp()
+        return self.path
+
+    def __exit__(self, exctype, value, traceback):
+        # rmtree in all cases
+        shutil.rmtree(self.path)
+        return traceback is None
+
+
+class pushd(object):
+    def __init__(self, directory):
+        self.directory = directory
+
+    def __enter__(self):
+        self.cwd = os.getcwd()
+        os.chdir(self.directory)
+        return self.directory
+
+    def __exit__(self, exctype, value, traceback):
+        os.chdir(self.cwd)
+
+
+def chown(path, login=None, group=None):
+    """Same as `os.chown` function but accepting user login or group name as
+    argument. If login or group is omitted, it's left unchanged.
+
+    Note: you must own the file to chown it (or be root). Otherwise OSError is raised.
+    """
+    if login is None:
+        uid = -1
+    else:
+        try:
+            uid = int(login)
+        except ValueError:
+            import pwd # Platforms: Unix
+            uid = pwd.getpwnam(login).pw_uid
+    if group is None:
+        gid = -1
+    else:
+        try:
+            gid = int(group)
+        except ValueError:
+            import grp
+            gid = grp.getgrnam(group).gr_gid
+    os.chown(path, uid, gid)
+
+def mv(source, destination, _action=shutil.move):
+    """A shell-like mv, supporting wildcards.
+    """
+    sources = glob.glob(source)
+    if len(sources) > 1:
+        assert isdir(destination)
+        for filename in sources:
+            _action(filename, join(destination, basename(filename)))
+    else:
+        try:
+            source = sources[0]
+        except IndexError:
+            raise OSError('No file matching %s' % source)
+        if isdir(destination) and exists(destination):
+            destination = join(destination, basename(source))
+        try:
+            _action(source, destination)
+        except OSError, ex:
+            raise OSError('Unable to move %r to %r (%s)' % (
+                source, destination, ex))
+
+def rm(*files):
+    """A shell-like rm, supporting wildcards.
+    """
+    for wfile in files:
+        for filename in glob.glob(wfile):
+            if islink(filename):
+                os.remove(filename)
+            elif isdir(filename):
+                shutil.rmtree(filename)
+            else:
+                os.remove(filename)
+
+def cp(source, destination):
+    """A shell-like cp, supporting wildcards.
+    """
+    mv(source, destination, _action=shutil.copy)
+
+def find(directory, exts, exclude=False, blacklist=STD_BLACKLIST):
+    """Recursively find files ending with the given extensions from the directory.
+
+    :type directory: str
+    :param directory:
+      directory where the search should start
+
+    :type exts: basestring or list or tuple
+    :param exts:
+      extensions or lists or extensions to search
+
+    :type exclude: boolean
+    :param exts:
+      if this argument is True, returning files NOT ending with the given
+      extensions
+
+    :type blacklist: list or tuple
+    :param blacklist:
+      optional list of files or directory to ignore, default to the value of
+      `logilab.common.STD_BLACKLIST`
+
+    :rtype: list
+    :return:
+      the list of all matching files
+    """
+    if isinstance(exts, basestring):
+        exts = (exts,)
+    if exclude:
+        def match(filename, exts):
+            for ext in exts:
+                if filename.endswith(ext):
+                    return False
+            return True
+    else:
+        def match(filename, exts):
+            for ext in exts:
+                if filename.endswith(ext):
+                    return True
+            return False
+    files = []
+    for dirpath, dirnames, filenames in os.walk(directory):
+        _handle_blacklist(blacklist, dirnames, filenames)
+        # don't append files if the directory is blacklisted
+        dirname = basename(dirpath)
+        if dirname in blacklist:
+            continue
+        files.extend([join(dirpath, f) for f in filenames if match(f, exts)])
+    return files
+
+
+def globfind(directory, pattern, blacklist=STD_BLACKLIST):
+    """Recursively finds files matching glob `pattern` under `directory`.
+
+    This is an alternative to `logilab.common.shellutils.find`.
+
+    :type directory: str
+    :param directory:
+      directory where the search should start
+
+    :type pattern: basestring
+    :param pattern:
+      the glob pattern (e.g *.py, foo*.py, etc.)
+
+    :type blacklist: list or tuple
+    :param blacklist:
+      optional list of files or directory to ignore, default to the value of
+      `logilab.common.STD_BLACKLIST`
+
+    :rtype: iterator
+    :return:
+      iterator over the list of all matching files
+    """
+    for curdir, dirnames, filenames in os.walk(directory):
+        _handle_blacklist(blacklist, dirnames, filenames)
+        for fname in fnmatch.filter(filenames, pattern):
+            yield join(curdir, fname)
+
+def unzip(archive, destdir):
+    import zipfile
+    if not exists(destdir):
+        os.mkdir(destdir)
+    zfobj = zipfile.ZipFile(archive)
+    for name in zfobj.namelist():
+        if name.endswith('/'):
+            os.mkdir(join(destdir, name))
+        else:
+            outfile = open(join(destdir, name), 'wb')
+            outfile.write(zfobj.read(name))
+            outfile.close()
+
+class Execute:
+    """This is a deadlock safe version of popen2 (no stdin), that returns
+    an object with errorlevel, out and err.
+    """
+
+    def __init__(self, command):
+        outfile = tempfile.mktemp()
+        errfile = tempfile.mktemp()
+        self.status = os.system("( %s ) >%s 2>%s" %
+                                (command, outfile, errfile)) >> 8
+        self.out = open(outfile, "r").read()
+        self.err = open(errfile, "r").read()
+        os.remove(outfile)
+        os.remove(errfile)
+
+def acquire_lock(lock_file, max_try=10, delay=10, max_delay=3600):
+    """Acquire a lock represented by a file on the file system
+
+    If the process written in lock file doesn't exist anymore, we remove the
+    lock file immediately
+    If age of the lock_file is greater than max_delay, then we raise a UserWarning
+    """
+    count = abs(max_try)
+    while count:
+        try:
+            fd = os.open(lock_file, os.O_EXCL | os.O_RDWR | os.O_CREAT)
+            os.write(fd, str_to_bytes(str(os.getpid())) )
+            os.close(fd)
+            return True
+        except OSError, e:
+            if e.errno == errno.EEXIST:
+                try:
+                    fd = open(lock_file, "r")
+                    pid = int(fd.readline())
+                    pi = ProcInfo(pid)
+                    age = (time.time() - os.stat(lock_file)[stat.ST_MTIME])
+                    if age / max_delay > 1 :
+                        raise UserWarning("Command '%s' (pid %s) has locked the "
+                                          "file '%s' for %s minutes"
+                                          % (pi.name(), pid, lock_file, age/60))
+                except UserWarning:
+                    raise
+                except NoSuchProcess:
+                    os.remove(lock_file)
+                except Exception:
+                    # The try block is not essential. can be skipped.
+                    # Note: ProcInfo object is only available for linux
+                    # process information are not accessible...
+                    # or lock_file is no more present...
+                    pass
+            else:
+                raise
+            count -= 1
+            time.sleep(delay)
+    else:
+        raise Exception('Unable to acquire %s' % lock_file)
+
+def release_lock(lock_file):
+    """Release a lock represented by a file on the file system."""
+    os.remove(lock_file)
+
+
+class ProgressBar(object):
+    """A simple text progression bar."""
+
+    def __init__(self, nbops, size=20, stream=sys.stdout, title=''):
+        if title:
+            self._fstr = '\r%s [%%-%ss]' % (title, int(size))
+        else:
+            self._fstr = '\r[%%-%ss]' % int(size)
+        self._stream = stream
+        self._total = nbops
+        self._size = size
+        self._current = 0
+        self._progress = 0
+        self._current_text = None
+        self._last_text_write_size = 0
+
+    def _get_text(self):
+        return self._current_text
+
+    def _set_text(self, text=None):
+        if text != self._current_text:
+            self._current_text = text
+            self.refresh()
+
+    def _del_text(self):
+        self.text = None
+
+    text = property(_get_text, _set_text, _del_text)
+
+    def update(self, offset=1, exact=False):
+        """Move FORWARD to new cursor position (cursor will never go backward).
+
+        :offset: fraction of ``size``
+
+        :exact:
+
+          - False: offset relative to current cursor position if True
+          - True: offset as an asbsolute position
+
+        """
+        if exact:
+            self._current = offset
+        else:
+            self._current += offset
+
+        progress = int((float(self._current)/float(self._total))*self._size)
+        if progress > self._progress:
+            self._progress = progress
+            self.refresh()
+
+    def refresh(self):
+        """Refresh the progression bar display."""
+        self._stream.write(self._fstr % ('.' * min(self._progress, self._size)) )
+        if self._last_text_write_size or self._current_text:
+            template = ' %%-%is' % (self._last_text_write_size)
+            text = self._current_text
+            if text is None:
+                text = ''
+            self._stream.write(template % text)
+            self._last_text_write_size = len(text.rstrip())
+        self._stream.flush()
+
+    def finish(self):
+        self._stream.write('\n')
+        self._stream.flush()
+
+
+class DummyProgressBar(object):
+    __slot__ = ('text',)
+
+    def refresh(self):
+        pass
+    def update(self):
+        pass
+    def finish(self):
+        pass
+
+
+_MARKER = object()
+class progress(object):
+
+    def __init__(self, nbops=_MARKER, size=_MARKER, stream=_MARKER, title=_MARKER, enabled=True):
+        self.nbops = nbops
+        self.size = size
+        self.stream = stream
+        self.title = title
+        self.enabled = enabled
+
+    def __enter__(self):
+        if self.enabled:
+            kwargs = {}
+            for attr in ('nbops', 'size', 'stream', 'title'):
+                value = getattr(self, attr)
+                if value is not _MARKER:
+                    kwargs[attr] = value
+            self.pb = ProgressBar(**kwargs)
+        else:
+            self.pb =  DummyProgressBar()
+        return self.pb
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.pb.finish()
+
+class RawInput(object):
+
+    def __init__(self, input=None, printer=None):
+        self._input = input or raw_input
+        self._print = printer
+
+    def ask(self, question, options, default):
+        assert default in options
+        choices = []
+        for option in options:
+            if option == default:
+                label = option[0].upper()
+            else:
+                label = option[0].lower()
+            if len(option) > 1:
+                label += '(%s)' % option[1:].lower()
+            choices.append((option, label))
+        prompt = "%s [%s]: " % (question,
+                                '/'.join([opt[1] for opt in choices]))
+        tries = 3
+        while tries > 0:
+            answer = self._input(prompt).strip().lower()
+            if not answer:
+                return default
+            possible = [option for option, label in choices
+                        if option.lower().startswith(answer)]
+            if len(possible) == 1:
+                return possible[0]
+            elif len(possible) == 0:
+                msg = '%s is not an option.' % answer
+            else:
+                msg = ('%s is an ambiguous answer, do you mean %s ?' % (
+                        answer, ' or '.join(possible)))
+            if self._print:
+                self._print(msg)
+            else:
+                print msg
+            tries -= 1
+        raise Exception('unable to get a sensible answer')
+
+    def confirm(self, question, default_is_yes=True):
+        default = default_is_yes and 'y' or 'n'
+        answer = self.ask(question, ('y', 'n'), default)
+        return answer == 'y'
+
+ASK = RawInput()
+
+
+def getlogin():
+    """avoid using os.getlogin() because of strange tty / stdin problems
+    (man 3 getlogin)
+    Another solution would be to use $LOGNAME, $USER or $USERNAME
+    """
+    if sys.platform != 'win32':
+        import pwd # Platforms: Unix
+        return pwd.getpwuid(os.getuid())[0]
+    else:
+        return os.environ['USERNAME']
+
+def generate_password(length=8, vocab=string.ascii_letters + string.digits):
+    """dumb password generation function"""
+    pwd = ''
+    for i in xrange(length):
+        pwd += random.choice(vocab)
+    return pwd

pylibs/logilab/common/sphinx_ext.py

@@ -0,0 +1,87 @@
+# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+#
+# This file is part of logilab-common.
+#
+# logilab-common is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation, either version 2.1 of the License, or (at your option) any
+# later version.
+#
+# logilab-common is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with logilab-common.  If not, see <http://www.gnu.org/licenses/>.
+from logilab.common.decorators import monkeypatch
+
+from sphinx.ext import autodoc
+
+class DocstringOnlyModuleDocumenter(autodoc.ModuleDocumenter):
+    objtype = 'docstring'
+    def format_signature(self):
+        pass
+    def add_directive_header(self, sig):
+        pass
+    def document_members(self, all_members=False):
+        pass
+
+    def resolve_name(self, modname, parents, path, base):
+        if modname is not None:
+            return modname, parents + [base]
+        return (path or '') + base, []
+
+
+#autodoc.add_documenter(DocstringOnlyModuleDocumenter)
+
+def setup(app):
+    app.add_autodocumenter(DocstringOnlyModuleDocumenter)
+
+
+
+from sphinx.ext.autodoc import (ViewList, Options, AutodocReporter, nodes,
+                                assemble_option_dict, nested_parse_with_titles)
+
+@monkeypatch(autodoc.AutoDirective)
+def run(self):
+    self.filename_set = set()  # a set of dependent filenames
+    self.reporter = self.state.document.reporter
+    self.env = self.state.document.settings.env
+    self.warnings = []
+    self.result = ViewList()
+
+    # find out what documenter to call
+    objtype = self.name[4:]
+    doc_class = self._registry[objtype]
+    # process the options with the selected documenter's option_spec
+    self.genopt = Options(assemble_option_dict(
+        self.options.items(), doc_class.option_spec))
+    # generate the output
+    documenter = doc_class(self, self.arguments[0])
+    documenter.generate(more_content=self.content)
+    if not self.result:
+        return self.warnings
+
+    # record all filenames as dependencies -- this will at least
+    # partially make automatic invalidation possible
+    for fn in self.filename_set:
+        self.env.note_dependency(fn)
+
+    # use a custom reporter that correctly assigns lines to source
+    # filename/description and lineno
+    old_reporter = self.state.memo.reporter
+    self.state.memo.reporter = AutodocReporter(self.result,
+                                               self.state.memo.reporter)
+    if self.name in ('automodule', 'autodocstring'):
+        node = nodes.section()
+        # necessary so that the child nodes get the right source/line set
+        node.document = self.state.document
+        nested_parse_with_titles(self.state, self.result, node)
+    else:
+        node = nodes.paragraph()
+        node.document = self.state.document
+        self.state.nested_parse(self.result, 0, node)
+    self.state.memo.reporter = old_reporter
+    return self.warnings + node.children

pylibs/logilab/common/sphinxutils.py

@@ -0,0 +1,122 @@
+# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+#
+# This file is part of logilab-common.
+#
+# logilab-common is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation, either version 2.1 of the License, or (at your option) any
+# later version.
+#
+# logilab-common is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with logilab-common.  If not, see <http://www.gnu.org/licenses/>.
+"""Sphinx utils
+
+ModuleGenerator: Generate a file that lists all the modules of a list of
+packages in order to pull all the docstring.
+This should not be used in a makefile to systematically generate sphinx
+documentation!
+
+Typical usage:
+
+>>> from logilab.common.sphinxutils import ModuleGenerator
+>>> mgen = ModuleGenerator('logilab common', '/home/adim/src/logilab/common')
+>>> mgen.generate('api_logilab_common.rst', exclude_dirs=('test',))
+"""
+
+import os, sys
+import os.path as osp
+import inspect
+
+from logilab.common import STD_BLACKLIST
+from logilab.common.shellutils import globfind
+from logilab.common.modutils import load_module_from_file, modpath_from_file
+
+def module_members(module):
+    members = []
+    for name, value in inspect.getmembers(module):
+        if getattr(value, '__module__', None) == module.__name__:
+            members.append( (name, value) )
+    return sorted(members)
+
+
+def class_members(klass):
+    return sorted([name for name in vars(klass)
+                   if name not in ('__doc__', '__module__',
+                                   '__dict__', '__weakref__')])
+
+class ModuleGenerator:
+    file_header = """.. -*- coding: utf-8 -*-\n\n%s\n"""
+    module_def = """
+:mod:`%s`
+=======%s
+
+.. automodule:: %s
+   :members: %s
+"""
+    class_def = """
+
+.. autoclass:: %s
+   :members: %s
+
+"""
+
+    def __init__(self, project_title, code_dir):
+        self.title = project_title
+        self.code_dir = osp.abspath(code_dir)
+
+    def generate(self, dest_file, exclude_dirs=STD_BLACKLIST):
+        """make the module file"""
+        self.fn = open(dest_file, 'w')
+        num = len(self.title) + 6
+        title = "=" * num + "\n %s API\n" % self.title + "=" * num
+        self.fn.write(self.file_header % title)
+        self.gen_modules(exclude_dirs=exclude_dirs)
+        self.fn.close()
+
+    def gen_modules(self, exclude_dirs):
+        """generate all modules"""
+        for module in self.find_modules(exclude_dirs):
+            modname = module.__name__
+            classes = []
+            modmembers = []
+            for objname, obj in module_members(module):
+                if inspect.isclass(obj):
+                    classmembers = class_members(obj)
+                    classes.append( (objname, classmembers) )
+                else:
+                    modmembers.append(objname)
+            self.fn.write(self.module_def % (modname, '=' * len(modname),
+                                             modname,
+                                             ', '.join(modmembers)))
+            for klass, members in classes:
+                self.fn.write(self.class_def % (klass, ', '.join(members)))
+
+    def find_modules(self, exclude_dirs):
+        basepath = osp.dirname(self.code_dir)
+        basedir = osp.basename(basepath) + osp.sep
+        if basedir not in sys.path:
+            sys.path.insert(1, basedir)
+        for filepath in globfind(self.code_dir, '*.py', exclude_dirs):
+            if osp.basename(filepath) in ('setup.py', '__pkginfo__.py'):
+                continue
+            try:
+                module = load_module_from_file(filepath)
+            except: # module might be broken or magic
+                dotted_path = modpath_from_file(filepath)
+                module = type('.'.join(dotted_path), (), {}) # mock it
+            yield module
+
+
+if __name__ == '__main__':
+    # example :
+    title, code_dir, outfile = sys.argv[1:]
+    generator = ModuleGenerator(title, code_dir)
+    # XXX modnames = ['logilab']
+    generator.generate(outfile, ('test', 'tests', 'examples',
+                             'data', 'doc', '.hg', 'migration'))

pylibs/logilab/common/table.py

@@ -0,0 +1,923 @@
+# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+#
+# This file is part of logilab-common.
+#
+# logilab-common is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation, either version 2.1 of the License, or (at your option) any
+# later version.
+#
+# logilab-common is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with logilab-common.  If not, see <http://www.gnu.org/licenses/>.
+"""Table management module."""
+__docformat__ = "restructuredtext en"
+
+
+class Table(object):
+    """Table defines a data table with column and row names.
+    inv:
+        len(self.data) <= len(self.row_names)
+        forall(self.data, lambda x: len(x) <= len(self.col_names))
+    """
+
+    def __init__(self, default_value=0, col_names=None, row_names=None):
+        self.col_names = []
+        self.row_names = []
+        self.data = []
+        self.default_value = default_value
+        if col_names:
+            self.create_columns(col_names)
+        if row_names:
+            self.create_rows(row_names)
+
+    def _next_row_name(self):
+        return 'row%s' % (len(self.row_names)+1)
+
+    def __iter__(self):
+        return iter(self.data)
+
+    def __eq__(self, other):
+        if other is None:
+            return False
+        else:
+            return list(self) == list(other)
+
+    def __ne__(self, other):
+        return not self == other
+
+    def __len__(self):
+        return len(self.row_names)
+
+    ## Rows / Columns creation #################################################
+    def create_rows(self, row_names):
+        """Appends row_names to the list of existing rows
+        """
+        self.row_names.extend(row_names)
+        for row_name in row_names:
+            self.data.append([self.default_value]*len(self.col_names))
+
+    def create_columns(self, col_names):
+        """Appends col_names to the list of existing columns
+        """
+        for col_name in col_names:
+            self.create_column(col_name)
+
+    def create_row(self, row_name=None):
+        """Creates a rowname to the row_names list
+        """
+        row_name = row_name or self._next_row_name()
+        self.row_names.append(row_name)
+        self.data.append([self.default_value]*len(self.col_names))
+
+
+    def create_column(self, col_name):
+        """Creates a colname to the col_names list
+        """
+        self.col_names.append(col_name)
+        for row in self.data:
+            row.append(self.default_value)
+
+    ## Sort by column ##########################################################
+    def sort_by_column_id(self, col_id, method = 'asc'):
+        """Sorts the table (in-place) according to data stored in col_id
+        """
+        try:
+            col_index = self.col_names.index(col_id)
+            self.sort_by_column_index(col_index, method)
+        except ValueError:
+            raise KeyError("Col (%s) not found in table" % (col_id))
+
+
+    def sort_by_column_index(self, col_index, method = 'asc'):
+        """Sorts the table 'in-place' according to data stored in col_index
+
+        method should be in ('asc', 'desc')
+        """
+        sort_list = sorted([(row[col_index], row, row_name)
+                     for row, row_name in zip(self.data, self.row_names)])
+        # Sorting sort_list will sort according to col_index
+        # If we want reverse sort, then reverse list
+        if method.lower() == 'desc':
+            sort_list.reverse()
+
+        # Rebuild data / row names
+        self.data = []
+        self.row_names = []
+        for val, row, row_name in sort_list:
+            self.data.append(row)
+            self.row_names.append(row_name)
+
+    def groupby(self, colname, *others):
+        """builds indexes of data
+        :returns: nested dictionaries pointing to actual rows
+        """
+        groups = {}
+        colnames = (colname,) + others
+        col_indexes = [self.col_names.index(col_id) for col_id in colnames]
+        for row in self.data:
+            ptr = groups
+            for col_index in col_indexes[:-1]:
+                ptr = ptr.setdefault(row[col_index], {})
+            ptr = ptr.setdefault(row[col_indexes[-1]],
+                                 Table(default_value=self.default_value,
+                                       col_names=self.col_names))
+            ptr.append_row(tuple(row))
+        return groups
+
+    def select(self, colname, value):
+        grouped = self.groupby(colname)
+        try:
+            return grouped[value]
+        except KeyError:
+            return []
+
+    def remove(self, colname, value):
+        col_index = self.col_names.index(colname)
+        for row in self.data[:]:
+            if row[col_index] == value:
+                self.data.remove(row)
+
+
+    ## The 'setter' part #######################################################
+    def set_cell(self, row_index, col_index, data):
+        """sets value of cell 'row_indew', 'col_index' to data
+        """
+        self.data[row_index][col_index] = data
+
+
+    def set_cell_by_ids(self, row_id, col_id, data):
+        """sets value of cell mapped by row_id and col_id to data
+        Raises a KeyError if row_id or col_id are not found in the table
+        """
+        try:
+            row_index = self.row_names.index(row_id)
+        except ValueError:
+            raise KeyError("Row (%s) not found in table" % (row_id))
+        else:
+            try:
+                col_index = self.col_names.index(col_id)
+                self.data[row_index][col_index] = data
+            except ValueError:
+                raise KeyError("Column (%s) not found in table" % (col_id))
+
+
+    def set_row(self, row_index, row_data):
+        """sets the 'row_index' row
+        pre:
+            type(row_data) == types.ListType
+            len(row_data) == len(self.col_names)
+        """
+        self.data[row_index] = row_data
+
+
+    def set_row_by_id(self, row_id, row_data):
+        """sets the 'row_id' column
+        pre:
+            type(row_data) == types.ListType
+            len(row_data) == len(self.row_names)
+        Raises a KeyError if row_id is not found
+        """
+        try:
+            row_index = self.row_names.index(row_id)
+            self.set_row(row_index, row_data)
+        except ValueError:
+            raise KeyError('Row (%s) not found in table' % (row_id))
+
+
+    def append_row(self, row_data, row_name=None):
+        """Appends a row to the table
+        pre:
+            type(row_data) == types.ListType
+            len(row_data) == len(self.col_names)
+        """
+        row_name = row_name or self._next_row_name()
+        self.row_names.append(row_name)
+        self.data.append(row_data)
+        return len(self.data) - 1
+
+    def insert_row(self, index, row_data, row_name=None):
+        """Appends row_data before 'index' in the table. To make 'insert'
+        behave like 'list.insert', inserting in an out of range index will
+        insert row_data to the end of the list
+        pre:
+            type(row_data) == types.ListType
+            len(row_data) == len(self.col_names)
+        """
+        row_name = row_name or self._next_row_name()
+        self.row_names.insert(index, row_name)
+        self.data.insert(index, row_data)
+
+
+    def delete_row(self, index):
+        """Deletes the 'index' row in the table, and returns it.
+        Raises an IndexError if index is out of range
+        """
+        self.row_names.pop(index)
+        return self.data.pop(index)
+
+
+    def delete_row_by_id(self, row_id):
+        """Deletes the 'row_id' row in the table.
+        Raises a KeyError if row_id was not found.
+        """
+        try:
+            row_index = self.row_names.index(row_id)
+            self.delete_row(row_index)
+        except ValueError:
+            raise KeyError('Row (%s) not found in table' % (row_id))
+
+
+    def set_column(self, col_index, col_data):
+        """sets the 'col_index' column
+        pre:
+            type(col_data) == types.ListType
+            len(col_data) == len(self.row_names)
+        """
+
+        for row_index, cell_data in enumerate(col_data):
+            self.data[row_index][col_index] = cell_data
+
+
+    def set_column_by_id(self, col_id, col_data):
+        """sets the 'col_id' column
+        pre:
+            type(col_data) == types.ListType
+            len(col_data) == len(self.col_names)
+        Raises a KeyError if col_id is not found
+        """
+        try:
+            col_index = self.col_names.index(col_id)
+            self.set_column(col_index, col_data)
+        except ValueError:
+            raise KeyError('Column (%s) not found in table' % (col_id))
+
+
+    def append_column(self, col_data, col_name):
+        """Appends the 'col_index' column
+        pre:
+            type(col_data) == types.ListType
+            len(col_data) == len(self.row_names)
+        """
+        self.col_names.append(col_name)
+        for row_index, cell_data in enumerate(col_data):
+            self.data[row_index].append(cell_data)
+
+
+    def insert_column(self, index, col_data, col_name):
+        """Appends col_data before 'index' in the table. To make 'insert'
+        behave like 'list.insert', inserting in an out of range index will
+        insert col_data to the end of the list
+        pre:
+            type(col_data) == types.ListType
+            len(col_data) == len(self.row_names)
+        """
+        self.col_names.insert(index, col_name)
+        for row_index, cell_data in enumerate(col_data):
+            self.data[row_index].insert(index, cell_data)
+
+
+    def delete_column(self, index):
+        """Deletes the 'index' column in the table, and returns it.
+        Raises an IndexError if index is out of range
+        """
+        self.col_names.pop(index)
+        return [row.pop(index) for row in self.data]
+
+
+    def delete_column_by_id(self, col_id):
+        """Deletes the 'col_id' col in the table.
+        Raises a KeyError if col_id was not found.
+        """
+        try:
+            col_index = self.col_names.index(col_id)
+            self.delete_column(col_index)
+        except ValueError:
+            raise KeyError('Column (%s) not found in table' % (col_id))
+
+
+    ## The 'getter' part #######################################################
+
+    def get_shape(self):
+        """Returns a tuple which represents the table's shape
+        """
+        return len(self.row_names), len(self.col_names)
+    shape = property(get_shape)
+
+    def __getitem__(self, indices):
+        """provided for convenience"""
+        rows, multirows = None, False
+        cols, multicols = None, False
+        if isinstance(indices, tuple):
+            rows = indices[0]
+            if len(indices) > 1:
+                cols = indices[1]
+        else:
+            rows = indices
+        # define row slice
+        if isinstance(rows, str):
+            try:
+                rows = self.row_names.index(rows)
+            except ValueError:
+                raise KeyError("Row (%s) not found in table" % (rows))
+        if isinstance(rows, int):
+            rows = slice(rows, rows+1)
+            multirows = False
+        else:
+            rows = slice(None)
+            multirows = True
+        # define col slice
+        if isinstance(cols, str):
+            try:
+                cols = self.col_names.index(cols)
+            except ValueError:
+                raise KeyError("Column (%s) not found in table" % (cols))
+        if isinstance(cols, int):
+            cols = slice(cols, cols+1)
+            multicols = False
+        else:
+            cols = slice(None)
+            multicols = True
+        # get sub-table
+        tab = Table()
+        tab.default_value = self.default_value
+        tab.create_rows(self.row_names[rows])
+        tab.create_columns(self.col_names[cols])
+        for idx, row in enumerate(self.data[rows]):
+            tab.set_row(idx, row[cols])
+        if multirows :
+            if multicols:
+                return tab
+            else:
+                return [item[0] for item in tab.data]
+        else:
+            if multicols:
+                return tab.data[0]
+            else:
+                return tab.data[0][0]
+
+    def get_cell_by_ids(self, row_id, col_id):
+        """Returns the element at [row_id][col_id]
+        """
+        try:
+            row_index = self.row_names.index(row_id)
+        except ValueError:
+            raise KeyError("Row (%s) not found in table" % (row_id))
+        else:
+            try:
+                col_index = self.col_names.index(col_id)
+            except ValueError:
+                raise KeyError("Column (%s) not found in table" % (col_id))
+        return self.data[row_index][col_index]
+
+    def get_row_by_id(self, row_id):
+        """Returns the 'row_id' row
+        """
+        try:
+            row_index = self.row_names.index(row_id)
+        except ValueError:
+            raise KeyError("Row (%s) not found in table" % (row_id))
+        return self.data[row_index]
+
+    def get_column_by_id(self, col_id, distinct=False):
+        """Returns the 'col_id' col
+        """
+        try:
+            col_index = self.col_names.index(col_id)
+        except ValueError:
+            raise KeyError("Column (%s) not found in table" % (col_id))
+        return self.get_column(col_index, distinct)
+
+    def get_columns(self):
+        """Returns all the columns in the table
+        """
+        return [self[:, index] for index in range(len(self.col_names))]
+
+    def get_column(self, col_index, distinct=False):
+        """get a column by index"""
+        col = [row[col_index] for row in self.data]
+        if distinct:
+            col = list(set(col))
+        return col
+
+    def apply_stylesheet(self, stylesheet):
+        """Applies the stylesheet to this table
+        """
+        for instruction in stylesheet.instructions:
+            eval(instruction)
+
+
+    def transpose(self):
+        """Keeps the self object intact, and returns the transposed (rotated)
+        table.
+        """
+        transposed = Table()
+        transposed.create_rows(self.col_names)
+        transposed.create_columns(self.row_names)
+        for col_index, column in enumerate(self.get_columns()):
+            transposed.set_row(col_index, column)
+        return transposed
+
+
+    def pprint(self):
+        """returns a string representing the table in a pretty
+        printed 'text' format.
+        """
+        # The maximum row name (to know the start_index of the first col)
+        max_row_name = 0
+        for row_name in self.row_names:
+            if len(row_name) > max_row_name:
+                max_row_name = len(row_name)
+        col_start = max_row_name + 5
+
+        lines = []
+        # Build the 'first' line <=> the col_names one
+        # The first cell <=> an empty one
+        col_names_line = [' '*col_start]
+        for col_name in self.col_names:
+            col_names_line.append(col_name.encode('iso-8859-1') + ' '*5)
+        lines.append('|' + '|'.join(col_names_line) + '|')
+        max_line_length = len(lines[0])
+
+        # Build the table
+        for row_index, row in enumerate(self.data):
+            line = []
+            # First, build the row_name's cell
+            row_name = self.row_names[row_index].encode('iso-8859-1')
+            line.append(row_name + ' '*(col_start-len(row_name)))
+
+            # Then, build all the table's cell for this line.
+            for col_index, cell in enumerate(row):
+                col_name_length = len(self.col_names[col_index]) + 5
+                data = str(cell)
+                line.append(data + ' '*(col_name_length - len(data)))
+            lines.append('|' + '|'.join(line) + '|')
+            if len(lines[-1]) > max_line_length:
+                max_line_length = len(lines[-1])
+
+        # Wrap the table with '-' to make a frame
+        lines.insert(0, '-'*max_line_length)
+        lines.append('-'*max_line_length)
+        return '\n'.join(lines)
+
+
+    def __repr__(self):
+        return repr(self.data)
+
+    def as_text(self):
+        data = []
+        # We must convert cells into strings before joining them
+        for row in self.data:
+            data.append([str(cell) for cell in row])
+        lines = ['\t'.join(row) for row in data]
+        return '\n'.join(lines)
+
+
+
+class TableStyle:
+    """Defines a table's style
+    """
+
+    def __init__(self, table):
+
+        self._table = table
+        self.size = dict([(col_name, '1*') for col_name in table.col_names])
+        # __row_column__ is a special key to define the first column which
+        # actually has no name (<=> left most column <=> row names column)
+        self.size['__row_column__'] = '1*'
+        self.alignment = dict([(col_name, 'right')
+                               for col_name in table.col_names])
+        self.alignment['__row_column__'] = 'right'
+
+        # We shouldn't have to create an entry for
+        # the 1st col (the row_column one)
+        self.units = dict([(col_name, '') for col_name in table.col_names])
+        self.units['__row_column__'] = ''
+
+    # XXX FIXME : params order should be reversed for all set() methods
+    def set_size(self, value, col_id):
+        """sets the size of the specified col_id to value
+        """
+        self.size[col_id] = value
+
+    def set_size_by_index(self, value, col_index):
+        """Allows to set the size according to the column index rather than
+        using the column's id.
+        BE CAREFUL : the '0' column is the '__row_column__' one !
+        """
+        if col_index == 0:
+            col_id = '__row_column__'
+        else:
+            col_id = self._table.col_names[col_index-1]
+
+        self.size[col_id] = value
+
+
+    def set_alignment(self, value, col_id):
+        """sets the alignment of the specified col_id to value
+        """
+        self.alignment[col_id] = value
+
+
+    def set_alignment_by_index(self, value, col_index):
+        """Allows to set the alignment according to the column index rather than
+        using the column's id.
+        BE CAREFUL : the '0' column is the '__row_column__' one !
+        """
+        if col_index == 0:
+            col_id = '__row_column__'
+        else:
+            col_id = self._table.col_names[col_index-1]
+
+        self.alignment[col_id] = value
+
+
+    def set_unit(self, value, col_id):
+        """sets the unit of the specified col_id to value
+        """
+        self.units[col_id] = value
+
+
+    def set_unit_by_index(self, value, col_index):
+        """Allows to set the unit according to the column index rather than
+        using the column's id.
+        BE CAREFUL :  the '0' column is the '__row_column__' one !
+        (Note that in the 'unit' case, you shouldn't have to set a unit
+        for the 1st column (the __row__column__ one))
+        """
+        if col_index == 0:
+            col_id = '__row_column__'
+        else:
+            col_id = self._table.col_names[col_index-1]
+
+        self.units[col_id] = value
+
+
+    def get_size(self, col_id):
+        """Returns the size of the specified col_id
+        """
+        return self.size[col_id]
+
+
+    def get_size_by_index(self, col_index):
+        """Allows to get the size  according to the column index rather than
+        using the column's id.
+        BE CAREFUL : the '0' column is the '__row_column__' one !
+        """
+        if col_index == 0:
+            col_id = '__row_column__'
+        else:
+            col_id = self._table.col_names[col_index-1]
+
+        return self.size[col_id]
+
+
+    def get_alignment(self, col_id):
+        """Returns the alignment of the specified col_id
+        """
+        return self.alignment[col_id]
+
+
+    def get_alignment_by_index(self, col_index):
+        """Allors to get the alignment according to the column index rather than
+        using the column's id.
+        BE CAREFUL : the '0' column is the '__row_column__' one !
+        """
+        if col_index == 0:
+            col_id = '__row_column__'
+        else:
+            col_id = self._table.col_names[col_index-1]
+
+        return self.alignment[col_id]
+
+
+    def get_unit(self, col_id):
+        """Returns the unit of the specified col_id
+        """
+        return self.units[col_id]
+
+
+    def get_unit_by_index(self, col_index):
+        """Allors to get the unit according to the column index rather than
+        using the column's id.
+        BE CAREFUL : the '0' column is the '__row_column__' one !
+        """
+        if col_index == 0:
+            col_id = '__row_column__'
+        else:
+            col_id = self._table.col_names[col_index-1]
+
+        return self.units[col_id]
+
+
+import re
+CELL_PROG = re.compile("([0-9]+)_([0-9]+)")
+
+class TableStyleSheet:
+    """A simple Table stylesheet
+    Rules are expressions where cells are defined by the row_index
+    and col_index separated by an underscore ('_').
+    For example, suppose you want to say that the (2,5) cell must be
+    the sum of its two preceding cells in the row, you would create
+    the following rule :
+        2_5 = 2_3 + 2_4
+    You can also use all the math.* operations you want. For example:
+        2_5 = sqrt(2_3**2 + 2_4**2)
+    """
+
+    def __init__(self, rules = None):
+        rules = rules or []
+        self.rules = []
+        self.instructions = []
+        for rule in rules:
+            self.add_rule(rule)
+
+
+    def add_rule(self, rule):
+        """Adds a rule to the stylesheet rules
+        """
+        try:
+            source_code = ['from math import *']
+            source_code.append(CELL_PROG.sub(r'self.data[\1][\2]', rule))
+            self.instructions.append(compile('\n'.join(source_code),
+                'table.py', 'exec'))
+            self.rules.append(rule)
+        except SyntaxError:
+            print "Bad Stylesheet Rule : %s [skipped]"%rule
+
+
+    def add_rowsum_rule(self, dest_cell, row_index, start_col, end_col):
+        """Creates and adds a rule to sum over the row at row_index from
+        start_col to end_col.
+        dest_cell is a tuple of two elements (x,y) of the destination cell
+        No check is done for indexes ranges.
+        pre:
+            start_col >= 0
+            end_col > start_col
+        """
+        cell_list = ['%d_%d'%(row_index, index) for index in range(start_col,
+                                                                   end_col + 1)]
+        rule = '%d_%d=' % dest_cell + '+'.join(cell_list)
+        self.add_rule(rule)
+
+
+    def add_rowavg_rule(self, dest_cell, row_index, start_col, end_col):
+        """Creates and adds a rule to make the row average (from start_col
+        to end_col)
+        dest_cell is a tuple of two elements (x,y) of the destination cell
+        No check is done for indexes ranges.
+        pre:
+            start_col >= 0
+            end_col > start_col
+        """
+        cell_list = ['%d_%d'%(row_index, index) for index in range(start_col,
+                                                                   end_col + 1)]
+        num = (end_col - start_col + 1)
+        rule = '%d_%d=' % dest_cell + '('+'+'.join(cell_list)+')/%f'%num
+        self.add_rule(rule)
+
+
+    def add_colsum_rule(self, dest_cell, col_index, start_row, end_row):
+        """Creates and adds a rule to sum over the col at col_index from
+        start_row to end_row.
+        dest_cell is a tuple of two elements (x,y) of the destination cell
+        No check is done for indexes ranges.
+        pre:
+            start_row >= 0
+            end_row > start_row
+        """
+        cell_list = ['%d_%d'%(index, col_index) for index in range(start_row,
+                                                                   end_row + 1)]
+        rule = '%d_%d=' % dest_cell + '+'.join(cell_list)
+        self.add_rule(rule)
+
+
+    def add_colavg_rule(self, dest_cell, col_index, start_row, end_row):
+        """Creates and adds a rule to make the col average (from start_row
+        to end_row)
+        dest_cell is a tuple of two elements (x,y) of the destination cell
+        No check is done for indexes ranges.
+        pre:
+            start_row >= 0
+            end_row > start_row
+        """
+        cell_list = ['%d_%d'%(index, col_index) for index in range(start_row,
+                                                                   end_row + 1)]
+        num = (end_row - start_row + 1)
+        rule = '%d_%d=' % dest_cell + '('+'+'.join(cell_list)+')/%f'%num
+        self.add_rule(rule)
+
+
+
+class TableCellRenderer:
+    """Defines a simple text renderer
+    """
+
+    def __init__(self, **properties):
+        """keywords should be properties with an associated boolean as value.
+        For example :
+            renderer = TableCellRenderer(units = True, alignment = False)
+        An unspecified property will have a 'False' value by default.
+        Possible properties are :
+            alignment, unit
+        """
+        self.properties = properties
+
+
+    def render_cell(self, cell_coord, table, table_style):
+        """Renders the cell at 'cell_coord' in the table, using table_style
+        """
+        row_index, col_index = cell_coord
+        cell_value = table.data[row_index][col_index]
+        final_content = self._make_cell_content(cell_value,
+                                                table_style, col_index  +1)
+        return self._render_cell_content(final_content,
+                                         table_style, col_index + 1)
+
+
+    def render_row_cell(self, row_name, table, table_style):
+        """Renders the cell for 'row_id' row
+        """
+        cell_value = row_name.encode('iso-8859-1')
+        return self._render_cell_content(cell_value, table_style, 0)
+
+
+    def render_col_cell(self, col_name, table, table_style):
+        """Renders the cell for 'col_id' row
+        """
+        cell_value = col_name.encode('iso-8859-1')
+        col_index = table.col_names.index(col_name)
+        return self._render_cell_content(cell_value, table_style, col_index +1)
+
+
+
+    def _render_cell_content(self, content, table_style, col_index):
+        """Makes the appropriate rendering for this cell content.
+        Rendering properties will be searched using the
+        *table_style.get_xxx_by_index(col_index)' methods
+
+        **This method should be overridden in the derived renderer classes.**
+        """
+        return content
+
+
+    def _make_cell_content(self, cell_content, table_style, col_index):
+        """Makes the cell content (adds decoration data, like units for
+        example)
+        """
+        final_content = cell_content
+        if 'skip_zero' in self.properties:
+            replacement_char = self.properties['skip_zero']
+        else:
+            replacement_char = 0
+        if replacement_char and final_content == 0:
+            return replacement_char
+
+        try:
+            units_on = self.properties['units']
+            if units_on:
+                final_content = self._add_unit(
+                    cell_content, table_style, col_index)
+        except KeyError:
+            pass
+
+        return final_content
+
+
+    def _add_unit(self, cell_content, table_style, col_index):
+        """Adds unit to the cell_content if needed
+        """
+        unit = table_style.get_unit_by_index(col_index)
+        return str(cell_content) + " " + unit
+
+
+
+class DocbookRenderer(TableCellRenderer):
+    """Defines how to render a cell for a docboook table
+    """
+
+    def define_col_header(self, col_index, table_style):
+        """Computes the colspec element according to the style
+        """
+        size = table_style.get_size_by_index(col_index)
+        return '<colspec colname="c%d" colwidth="%s"/>\n' % \
+               (col_index, size)
+
+
+    def _render_cell_content(self, cell_content, table_style, col_index):
+        """Makes the appropriate rendering for this cell content.
+        Rendering properties will be searched using the
+        table_style.get_xxx_by_index(col_index)' methods.
+        """
+        try:
+            align_on = self.properties['alignment']
+            alignment = table_style.get_alignment_by_index(col_index)
+            if align_on:
+                return "<entry align='%s'>%s</entry>\n" % \
+                       (alignment, cell_content)
+        except KeyError:
+            # KeyError <=> Default alignment
+            return "<entry>%s</entry>\n" % cell_content
+
+
+class TableWriter:
+    """A class to write tables
+    """
+
+    def __init__(self, stream, table, style, **properties):
+        self._stream = stream
+        self.style = style or TableStyle(table)
+        self._table = table
+        self.properties = properties
+        self.renderer = None
+
+
+    def set_style(self, style):
+        """sets the table's associated style
+        """
+        self.style = style
+
+
+    def set_renderer(self, renderer):
+        """sets the way to render cell
+        """
+        self.renderer = renderer
+
+
+    def update_properties(self, **properties):
+        """Updates writer's properties (for cell rendering)
+        """
+        self.properties.update(properties)
+
+
+    def write_table(self, title = ""):
+        """Writes the table
+        """
+        raise NotImplementedError("write_table must be implemented !")
+
+
+
+class DocbookTableWriter(TableWriter):
+    """Defines an implementation of TableWriter to write a table in Docbook
+    """
+
+    def _write_headers(self):
+        """Writes col headers
+        """
+        # Define col_headers (colstpec elements)
+        for col_index in range(len(self._table.col_names)+1):
+            self._stream.write(self.renderer.define_col_header(col_index,
+                                                              self.style))
+
+        self._stream.write("<thead>\n<row>\n")
+        # XXX FIXME : write an empty entry <=> the first (__row_column) column
+        self._stream.write('<entry></entry>\n')
+        for col_name in self._table.col_names:
+            self._stream.write(self.renderer.render_col_cell(
+                col_name, self._table,
+                self.style))
+
+        self._stream.write("</row>\n</thead>\n")
+
+
+    def _write_body(self):
+        """Writes the table body
+        """
+        self._stream.write('<tbody>\n')
+
+        for row_index, row in enumerate(self._table.data):
+            self._stream.write('<row>\n')
+            row_name = self._table.row_names[row_index]
+            # Write the first entry (row_name)
+            self._stream.write(self.renderer.render_row_cell(row_name,
+                                                            self._table,
+                                                            self.style))
+
+            for col_index, cell in enumerate(row):
+                self._stream.write(self.renderer.render_cell(
+                    (row_index, col_index),
+                    self._table, self.style))
+
+            self._stream.write('</row>\n')
+
+        self._stream.write('</tbody>\n')
+
+
+    def write_table(self, title = ""):
+        """Writes the table
+        """
+        self._stream.write('<table>\n<title>%s></title>\n'%(title))
+        self._stream.write(
+            '<tgroup cols="%d" align="left" colsep="1" rowsep="1">\n'%
+            (len(self._table.col_names)+1))
+        self._write_headers()
+        self._write_body()
+
+        self._stream.write('</tgroup>\n</table>\n')
+
+