Initial commit
This commit is contained in:
retanoj
0
mosec/__init__.py
Normal file
0
mosec/__init__.py
Normal file
27
mosec/colorama/LICENSE.txt
Normal file
27
mosec/colorama/LICENSE.txt
Normal file
@@ -0,0 +1,27 @@
|
||||
Copyright (c) 2010 Jonathan Hartley
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright notice, this
|
||||
list of conditions and the following disclaimer.
|
||||
|
||||
* Redistributions in binary form must reproduce the above copyright notice,
|
||||
this list of conditions and the following disclaimer in the documentation
|
||||
and/or other materials provided with the distribution.
|
||||
|
||||
* Neither the name of the copyright holders, nor those of its contributors
|
||||
may be used to endorse or promote products derived from this software without
|
||||
specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
380
mosec/colorama/README.rst
Normal file
380
mosec/colorama/README.rst
Normal file
@@ -0,0 +1,380 @@
|
||||
.. image:: https://img.shields.io/pypi/v/colorama.svg
|
||||
:target: https://pypi.org/project/colorama/
|
||||
:alt: Latest Version
|
||||
|
||||
.. image:: https://img.shields.io/pypi/pyversions/colorama.svg
|
||||
:target: https://pypi.org/project/colorama/
|
||||
:alt: Supported Python versions
|
||||
|
||||
.. image:: https://travis-ci.org/tartley/colorama.svg?branch=master
|
||||
:target: https://travis-ci.org/tartley/colorama
|
||||
:alt: Build Status
|
||||
|
||||
Download and docs:
|
||||
https://pypi.org/project/colorama/
|
||||
Source code & Development:
|
||||
https://github.com/tartley/colorama
|
||||
Colorama for Enterprise:
|
||||
https://github.com/tartley/colorama/blob/master/ENTERPRISE.md
|
||||
|
||||
Description
|
||||
===========
|
||||
|
||||
Makes ANSI escape character sequences (for producing colored terminal text and
|
||||
cursor positioning) work under MS Windows.
|
||||
|
||||
ANSI escape character sequences have long been used to produce colored terminal
|
||||
text and cursor positioning on Unix and Macs. Colorama makes this work on
|
||||
Windows, too, by wrapping ``stdout``, stripping ANSI sequences it finds (which
|
||||
would appear as gobbledygook in the output), and converting them into the
|
||||
appropriate win32 calls to modify the state of the terminal. On other platforms,
|
||||
Colorama does nothing.
|
||||
|
||||
Colorama also provides some shortcuts to help generate ANSI sequences
|
||||
but works fine in conjunction with any other ANSI sequence generation library,
|
||||
such as the venerable Termcolor (https://pypi.org/project/termcolor/)
|
||||
or the fabulous Blessings (https://pypi.org/project/blessings/).
|
||||
|
||||
This has the upshot of providing a simple cross-platform API for printing
|
||||
colored terminal text from Python, and has the happy side-effect that existing
|
||||
applications or libraries which use ANSI sequences to produce colored output on
|
||||
Linux or Macs can now also work on Windows, simply by calling
|
||||
``colorama.init()``.
|
||||
|
||||
An alternative approach is to install ``ansi.sys`` on Windows machines, which
|
||||
provides the same behaviour for all applications running in terminals. Colorama
|
||||
is intended for situations where that isn't easy (e.g., maybe your app doesn't
|
||||
have an installer.)
|
||||
|
||||
Demo scripts in the source code repository print some colored text using
|
||||
ANSI sequences. Compare their output under Gnome-terminal's built in ANSI
|
||||
handling, versus on Windows Command-Prompt using Colorama:
|
||||
|
||||
.. image:: https://github.com/tartley/colorama/raw/master/screenshots/ubuntu-demo.png
|
||||
:width: 661
|
||||
:height: 357
|
||||
:alt: ANSI sequences on Ubuntu under gnome-terminal.
|
||||
|
||||
.. image:: https://github.com/tartley/colorama/raw/master/screenshots/windows-demo.png
|
||||
:width: 668
|
||||
:height: 325
|
||||
:alt: Same ANSI sequences on Windows, using Colorama.
|
||||
|
||||
These screengrabs show that, on Windows, Colorama does not support ANSI 'dim
|
||||
text'; it looks the same as 'normal text'.
|
||||
|
||||
|
||||
License
|
||||
=======
|
||||
|
||||
Copyright Jonathan Hartley & Arnon Yaari, 2013. BSD 3-Clause license; see LICENSE file.
|
||||
|
||||
|
||||
Dependencies
|
||||
============
|
||||
|
||||
None, other than Python. Tested on Python 2.7, 3.5, 3.6, 3.7 and 3.8.
|
||||
|
||||
Usage
|
||||
=====
|
||||
|
||||
Initialisation
|
||||
--------------
|
||||
|
||||
Applications should initialise Colorama using:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
from colorama import init
|
||||
init()
|
||||
|
||||
On Windows, calling ``init()`` will filter ANSI escape sequences out of any
|
||||
text sent to ``stdout`` or ``stderr``, and replace them with equivalent Win32
|
||||
calls.
|
||||
|
||||
On other platforms, calling ``init()`` has no effect (unless you request other
|
||||
optional functionality; see "Init Keyword Args", below). By design, this permits
|
||||
applications to call ``init()`` unconditionally on all platforms, after which
|
||||
ANSI output should just work.
|
||||
|
||||
To stop using colorama before your program exits, simply call ``deinit()``.
|
||||
This will restore ``stdout`` and ``stderr`` to their original values, so that
|
||||
Colorama is disabled. To resume using Colorama again, call ``reinit()``; it is
|
||||
cheaper than calling ``init()`` again (but does the same thing).
|
||||
|
||||
|
||||
Colored Output
|
||||
--------------
|
||||
|
||||
Cross-platform printing of colored text can then be done using Colorama's
|
||||
constant shorthand for ANSI escape sequences:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
from colorama import Fore, Back, Style
|
||||
print(Fore.RED + 'some red text')
|
||||
print(Back.GREEN + 'and with a green background')
|
||||
print(Style.DIM + 'and in dim text')
|
||||
print(Style.RESET_ALL)
|
||||
print('back to normal now')
|
||||
|
||||
...or simply by manually printing ANSI sequences from your own code:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
print('\033[31m' + 'some red text')
|
||||
print('\033[39m') # and reset to default color
|
||||
|
||||
...or, Colorama can be used happily in conjunction with existing ANSI libraries
|
||||
such as Termcolor:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
from colorama import init
|
||||
from termcolor import colored
|
||||
|
||||
# use Colorama to make Termcolor work on Windows too
|
||||
init()
|
||||
|
||||
# then use Termcolor for all colored text output
|
||||
print(colored('Hello, World!', 'green', 'on_red'))
|
||||
|
||||
Available formatting constants are::
|
||||
|
||||
Fore: BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE, RESET.
|
||||
Back: BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE, RESET.
|
||||
Style: DIM, NORMAL, BRIGHT, RESET_ALL
|
||||
|
||||
``Style.RESET_ALL`` resets foreground, background, and brightness. Colorama will
|
||||
perform this reset automatically on program exit.
|
||||
|
||||
|
||||
Cursor Positioning
|
||||
------------------
|
||||
|
||||
ANSI codes to reposition the cursor are supported. See ``demos/demo06.py`` for
|
||||
an example of how to generate them.
|
||||
|
||||
|
||||
Init Keyword Args
|
||||
-----------------
|
||||
|
||||
``init()`` accepts some ``**kwargs`` to override default behaviour.
|
||||
|
||||
init(autoreset=False):
|
||||
If you find yourself repeatedly sending reset sequences to turn off color
|
||||
changes at the end of every print, then ``init(autoreset=True)`` will
|
||||
automate that:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
from colorama import init
|
||||
init(autoreset=True)
|
||||
print(Fore.RED + 'some red text')
|
||||
print('automatically back to default color again')
|
||||
|
||||
init(strip=None):
|
||||
Pass ``True`` or ``False`` to override whether ansi codes should be
|
||||
stripped from the output. The default behaviour is to strip if on Windows
|
||||
or if output is redirected (not a tty).
|
||||
|
||||
init(convert=None):
|
||||
Pass ``True`` or ``False`` to override whether to convert ANSI codes in the
|
||||
output into win32 calls. The default behaviour is to convert if on Windows
|
||||
and output is to a tty (terminal).
|
||||
|
||||
init(wrap=True):
|
||||
On Windows, colorama works by replacing ``sys.stdout`` and ``sys.stderr``
|
||||
with proxy objects, which override the ``.write()`` method to do their work.
|
||||
If this wrapping causes you problems, then this can be disabled by passing
|
||||
``init(wrap=False)``. The default behaviour is to wrap if ``autoreset`` or
|
||||
``strip`` or ``convert`` are True.
|
||||
|
||||
When wrapping is disabled, colored printing on non-Windows platforms will
|
||||
continue to work as normal. To do cross-platform colored output, you can
|
||||
use Colorama's ``AnsiToWin32`` proxy directly:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
import sys
|
||||
from colorama import init, AnsiToWin32
|
||||
init(wrap=False)
|
||||
stream = AnsiToWin32(sys.stderr).stream
|
||||
|
||||
# Python 2
|
||||
print >>stream, Fore.BLUE + 'blue text on stderr'
|
||||
|
||||
# Python 3
|
||||
print(Fore.BLUE + 'blue text on stderr', file=stream)
|
||||
|
||||
|
||||
Installation
|
||||
=======================
|
||||
colorama is currently installable from PyPI:
|
||||
|
||||
pip install colorama
|
||||
|
||||
colorama also can be installed by the conda package manager:
|
||||
|
||||
conda install -c anaconda colorama
|
||||
|
||||
|
||||
Status & Known Problems
|
||||
=======================
|
||||
|
||||
I've personally only tested it on Windows XP (CMD, Console2), Ubuntu
|
||||
(gnome-terminal, xterm), and OS X.
|
||||
|
||||
Some presumably valid ANSI sequences aren't recognised (see details below),
|
||||
but to my knowledge nobody has yet complained about this. Puzzling.
|
||||
|
||||
See outstanding issues and wishlist:
|
||||
https://github.com/tartley/colorama/issues
|
||||
|
||||
If anything doesn't work for you, or doesn't do what you expected or hoped for,
|
||||
I'd love to hear about it on that issues list, would be delighted by patches,
|
||||
and would be happy to grant commit access to anyone who submits a working patch
|
||||
or two.
|
||||
|
||||
|
||||
Recognised ANSI Sequences
|
||||
=========================
|
||||
|
||||
ANSI sequences generally take the form:
|
||||
|
||||
ESC [ <param> ; <param> ... <command>
|
||||
|
||||
Where ``<param>`` is an integer, and ``<command>`` is a single letter. Zero or
|
||||
more params are passed to a ``<command>``. If no params are passed, it is
|
||||
generally synonymous with passing a single zero. No spaces exist in the
|
||||
sequence; they have been inserted here simply to read more easily.
|
||||
|
||||
The only ANSI sequences that colorama converts into win32 calls are::
|
||||
|
||||
ESC [ 0 m # reset all (colors and brightness)
|
||||
ESC [ 1 m # bright
|
||||
ESC [ 2 m # dim (looks same as normal brightness)
|
||||
ESC [ 22 m # normal brightness
|
||||
|
||||
# FOREGROUND:
|
||||
ESC [ 30 m # black
|
||||
ESC [ 31 m # red
|
||||
ESC [ 32 m # green
|
||||
ESC [ 33 m # yellow
|
||||
ESC [ 34 m # blue
|
||||
ESC [ 35 m # magenta
|
||||
ESC [ 36 m # cyan
|
||||
ESC [ 37 m # white
|
||||
ESC [ 39 m # reset
|
||||
|
||||
# BACKGROUND
|
||||
ESC [ 40 m # black
|
||||
ESC [ 41 m # red
|
||||
ESC [ 42 m # green
|
||||
ESC [ 43 m # yellow
|
||||
ESC [ 44 m # blue
|
||||
ESC [ 45 m # magenta
|
||||
ESC [ 46 m # cyan
|
||||
ESC [ 47 m # white
|
||||
ESC [ 49 m # reset
|
||||
|
||||
# cursor positioning
|
||||
ESC [ y;x H # position cursor at x across, y down
|
||||
ESC [ y;x f # position cursor at x across, y down
|
||||
ESC [ n A # move cursor n lines up
|
||||
ESC [ n B # move cursor n lines down
|
||||
ESC [ n C # move cursor n characters forward
|
||||
ESC [ n D # move cursor n characters backward
|
||||
|
||||
# clear the screen
|
||||
ESC [ mode J # clear the screen
|
||||
|
||||
# clear the line
|
||||
ESC [ mode K # clear the line
|
||||
|
||||
Multiple numeric params to the ``'m'`` command can be combined into a single
|
||||
sequence::
|
||||
|
||||
ESC [ 36 ; 45 ; 1 m # bright cyan text on magenta background
|
||||
|
||||
All other ANSI sequences of the form ``ESC [ <param> ; <param> ... <command>``
|
||||
are silently stripped from the output on Windows.
|
||||
|
||||
Any other form of ANSI sequence, such as single-character codes or alternative
|
||||
initial characters, are not recognised or stripped. It would be cool to add
|
||||
them though. Let me know if it would be useful for you, via the Issues on
|
||||
GitHub.
|
||||
|
||||
|
||||
Development
|
||||
===========
|
||||
|
||||
Help and fixes welcome!
|
||||
|
||||
Running tests requires:
|
||||
|
||||
- Michael Foord's ``mock`` module to be installed on Python < 3.3.
|
||||
- Tests are written using 2010-era updates to ``unittest``
|
||||
|
||||
To run tests::
|
||||
|
||||
python -m unittest discover -p *_test.py
|
||||
|
||||
This, like a few other handy commands, is captured in a ``Makefile``.
|
||||
|
||||
If you use nose to run the tests, you must pass the ``-s`` flag; otherwise,
|
||||
``nosetests`` applies its own proxy to ``stdout``, which confuses the unit
|
||||
tests.
|
||||
|
||||
|
||||
Professional support
|
||||
====================
|
||||
|
||||
.. |tideliftlogo| image:: https://cdn2.hubspot.net/hubfs/4008838/website/logos/logos_for_download/Tidelift_primary-shorthand-logo.png
|
||||
:alt: Tidelift
|
||||
:target: https://tidelift.com/subscription/pkg/pypi-colorama?utm_source=pypi-colorama&utm_medium=referral&utm_campaign=readme
|
||||
|
||||
.. list-table::
|
||||
:widths: 10 100
|
||||
|
||||
* - |tideliftlogo|
|
||||
- Professional support for colorama is available as part of the
|
||||
`Tidelift Subscription`_.
|
||||
Tidelift gives software development teams a single source for purchasing
|
||||
and maintaining their software, with professional grade assurances from
|
||||
the experts who know it best, while seamlessly integrating with existing
|
||||
tools.
|
||||
|
||||
.. _Tidelift Subscription: https://tidelift.com/subscription/pkg/pypi-colorama?utm_source=pypi-colorama&utm_medium=referral&utm_campaign=readme
|
||||
|
||||
|
||||
Thanks
|
||||
======
|
||||
* Marc Schlaich (schlamar) for a ``setup.py`` fix for Python2.5.
|
||||
* Marc Abramowitz, reported & fixed a crash on exit with closed ``stdout``,
|
||||
providing a solution to issue #7's setuptools/distutils debate,
|
||||
and other fixes.
|
||||
* User 'eryksun', for guidance on correctly instantiating ``ctypes.windll``.
|
||||
* Matthew McCormick for politely pointing out a longstanding crash on non-Win.
|
||||
* Ben Hoyt, for a magnificent fix under 64-bit Windows.
|
||||
* Jesse at Empty Square for submitting a fix for examples in the README.
|
||||
* User 'jamessp', an observant documentation fix for cursor positioning.
|
||||
* User 'vaal1239', Dave Mckee & Lackner Kristof for a tiny but much-needed Win7
|
||||
fix.
|
||||
* Julien Stuyck, for wisely suggesting Python3 compatible updates to README.
|
||||
* Daniel Griffith for multiple fabulous patches.
|
||||
* Oscar Lesta for a valuable fix to stop ANSI chars being sent to non-tty
|
||||
output.
|
||||
* Roger Binns, for many suggestions, valuable feedback, & bug reports.
|
||||
* Tim Golden for thought and much appreciated feedback on the initial idea.
|
||||
* User 'Zearin' for updates to the README file.
|
||||
* John Szakmeister for adding support for light colors
|
||||
* Charles Merriam for adding documentation to demos
|
||||
* Jurko for a fix on 64-bit Windows CPython2.5 w/o ctypes
|
||||
* Florian Bruhin for a fix when stdout or stderr are None
|
||||
* Thomas Weininger for fixing ValueError on Windows
|
||||
* Remi Rampin for better Github integration and fixes to the README file
|
||||
* Simeon Visser for closing a file handle using 'with' and updating classifiers
|
||||
to include Python 3.3 and 3.4
|
||||
* Andy Neff for fixing RESET of LIGHT_EX colors.
|
||||
* Jonathan Hartley for the initial idea and implementation.
|
||||
6
mosec/colorama/__init__.py
Normal file
6
mosec/colorama/__init__.py
Normal file
@@ -0,0 +1,6 @@
|
||||
# Copyright Jonathan Hartley 2013. BSD 3-Clause license, see LICENSE file.
|
||||
from .initialise import init, deinit, reinit, colorama_text
|
||||
from .ansi import Fore, Back, Style, Cursor
|
||||
from .ansitowin32 import AnsiToWin32
|
||||
|
||||
__version__ = '0.4.4'
|
||||
102
mosec/colorama/ansi.py
Normal file
102
mosec/colorama/ansi.py
Normal file
@@ -0,0 +1,102 @@
|
||||
# Copyright Jonathan Hartley 2013. BSD 3-Clause license, see LICENSE file.
|
||||
'''
|
||||
This module generates ANSI character codes to printing colors to terminals.
|
||||
See: http://en.wikipedia.org/wiki/ANSI_escape_code
|
||||
'''
|
||||
|
||||
CSI = '\033['
|
||||
OSC = '\033]'
|
||||
BEL = '\a'
|
||||
|
||||
|
||||
def code_to_chars(code):
|
||||
return CSI + str(code) + 'm'
|
||||
|
||||
def set_title(title):
|
||||
return OSC + '2;' + title + BEL
|
||||
|
||||
def clear_screen(mode=2):
|
||||
return CSI + str(mode) + 'J'
|
||||
|
||||
def clear_line(mode=2):
|
||||
return CSI + str(mode) + 'K'
|
||||
|
||||
|
||||
class AnsiCodes(object):
|
||||
def __init__(self):
|
||||
# the subclasses declare class attributes which are numbers.
|
||||
# Upon instantiation we define instance attributes, which are the same
|
||||
# as the class attributes but wrapped with the ANSI escape sequence
|
||||
for name in dir(self):
|
||||
if not name.startswith('_'):
|
||||
value = getattr(self, name)
|
||||
setattr(self, name, code_to_chars(value))
|
||||
|
||||
|
||||
class AnsiCursor(object):
|
||||
def UP(self, n=1):
|
||||
return CSI + str(n) + 'A'
|
||||
def DOWN(self, n=1):
|
||||
return CSI + str(n) + 'B'
|
||||
def FORWARD(self, n=1):
|
||||
return CSI + str(n) + 'C'
|
||||
def BACK(self, n=1):
|
||||
return CSI + str(n) + 'D'
|
||||
def POS(self, x=1, y=1):
|
||||
return CSI + str(y) + ';' + str(x) + 'H'
|
||||
|
||||
|
||||
class AnsiFore(AnsiCodes):
|
||||
BLACK = 30
|
||||
RED = 31
|
||||
GREEN = 32
|
||||
YELLOW = 33
|
||||
BLUE = 34
|
||||
MAGENTA = 35
|
||||
CYAN = 36
|
||||
WHITE = 37
|
||||
RESET = 39
|
||||
|
||||
# These are fairly well supported, but not part of the standard.
|
||||
LIGHTBLACK_EX = 90
|
||||
LIGHTRED_EX = 91
|
||||
LIGHTGREEN_EX = 92
|
||||
LIGHTYELLOW_EX = 93
|
||||
LIGHTBLUE_EX = 94
|
||||
LIGHTMAGENTA_EX = 95
|
||||
LIGHTCYAN_EX = 96
|
||||
LIGHTWHITE_EX = 97
|
||||
|
||||
|
||||
class AnsiBack(AnsiCodes):
|
||||
BLACK = 40
|
||||
RED = 41
|
||||
GREEN = 42
|
||||
YELLOW = 43
|
||||
BLUE = 44
|
||||
MAGENTA = 45
|
||||
CYAN = 46
|
||||
WHITE = 47
|
||||
RESET = 49
|
||||
|
||||
# These are fairly well supported, but not part of the standard.
|
||||
LIGHTBLACK_EX = 100
|
||||
LIGHTRED_EX = 101
|
||||
LIGHTGREEN_EX = 102
|
||||
LIGHTYELLOW_EX = 103
|
||||
LIGHTBLUE_EX = 104
|
||||
LIGHTMAGENTA_EX = 105
|
||||
LIGHTCYAN_EX = 106
|
||||
LIGHTWHITE_EX = 107
|
||||
|
||||
|
||||
class AnsiStyle(AnsiCodes):
|
||||
BRIGHT = 1
|
||||
DIM = 2
|
||||
NORMAL = 22
|
||||
RESET_ALL = 0
|
||||
|
||||
Fore = AnsiFore()
|
||||
Back = AnsiBack()
|
||||
Style = AnsiStyle()
|
||||
Cursor = AnsiCursor()
|
||||
258
mosec/colorama/ansitowin32.py
Normal file
258
mosec/colorama/ansitowin32.py
Normal file
@@ -0,0 +1,258 @@
|
||||
# Copyright Jonathan Hartley 2013. BSD 3-Clause license, see LICENSE file.
|
||||
import re
|
||||
import sys
|
||||
import os
|
||||
|
||||
from .ansi import AnsiFore, AnsiBack, AnsiStyle, Style, BEL
|
||||
from .winterm import WinTerm, WinColor, WinStyle
|
||||
from .win32 import windll, winapi_test
|
||||
|
||||
|
||||
winterm = None
|
||||
if windll is not None:
|
||||
winterm = WinTerm()
|
||||
|
||||
|
||||
class StreamWrapper(object):
|
||||
'''
|
||||
Wraps a stream (such as stdout), acting as a transparent proxy for all
|
||||
attribute access apart from method 'write()', which is delegated to our
|
||||
Converter instance.
|
||||
'''
|
||||
def __init__(self, wrapped, converter):
|
||||
# double-underscore everything to prevent clashes with names of
|
||||
# attributes on the wrapped stream object.
|
||||
self.__wrapped = wrapped
|
||||
self.__convertor = converter
|
||||
|
||||
def __getattr__(self, name):
|
||||
return getattr(self.__wrapped, name)
|
||||
|
||||
def __enter__(self, *args, **kwargs):
|
||||
# special method lookup bypasses __getattr__/__getattribute__, see
|
||||
# https://stackoverflow.com/questions/12632894/why-doesnt-getattr-work-with-exit
|
||||
# thus, contextlib magic methods are not proxied via __getattr__
|
||||
return self.__wrapped.__enter__(*args, **kwargs)
|
||||
|
||||
def __exit__(self, *args, **kwargs):
|
||||
return self.__wrapped.__exit__(*args, **kwargs)
|
||||
|
||||
def write(self, text):
|
||||
self.__convertor.write(text)
|
||||
|
||||
def isatty(self):
|
||||
stream = self.__wrapped
|
||||
if 'PYCHARM_HOSTED' in os.environ:
|
||||
if stream is not None and (stream is sys.__stdout__ or stream is sys.__stderr__):
|
||||
return True
|
||||
try:
|
||||
stream_isatty = stream.isatty
|
||||
except AttributeError:
|
||||
return False
|
||||
else:
|
||||
return stream_isatty()
|
||||
|
||||
@property
|
||||
def closed(self):
|
||||
stream = self.__wrapped
|
||||
try:
|
||||
return stream.closed
|
||||
except AttributeError:
|
||||
return True
|
||||
|
||||
|
||||
class AnsiToWin32(object):
|
||||
'''
|
||||
Implements a 'write()' method which, on Windows, will strip ANSI character
|
||||
sequences from the text, and if outputting to a tty, will convert them into
|
||||
win32 function calls.
|
||||
'''
|
||||
ANSI_CSI_RE = re.compile('\001?\033\\[((?:\\d|;)*)([a-zA-Z])\002?') # Control Sequence Introducer
|
||||
ANSI_OSC_RE = re.compile('\001?\033\\]([^\a]*)(\a)\002?') # Operating System Command
|
||||
|
||||
def __init__(self, wrapped, convert=None, strip=None, autoreset=False):
|
||||
# The wrapped stream (normally sys.stdout or sys.stderr)
|
||||
self.wrapped = wrapped
|
||||
|
||||
# should we reset colors to defaults after every .write()
|
||||
self.autoreset = autoreset
|
||||
|
||||
# create the proxy wrapping our output stream
|
||||
self.stream = StreamWrapper(wrapped, self)
|
||||
|
||||
on_windows = os.name == 'nt'
|
||||
# We test if the WinAPI works, because even if we are on Windows
|
||||
# we may be using a terminal that doesn't support the WinAPI
|
||||
# (e.g. Cygwin Terminal). In this case it's up to the terminal
|
||||
# to support the ANSI codes.
|
||||
conversion_supported = on_windows and winapi_test()
|
||||
|
||||
# should we strip ANSI sequences from our output?
|
||||
if strip is None:
|
||||
strip = conversion_supported or (not self.stream.closed and not self.stream.isatty())
|
||||
self.strip = strip
|
||||
|
||||
# should we should convert ANSI sequences into win32 calls?
|
||||
if convert is None:
|
||||
convert = conversion_supported and not self.stream.closed and self.stream.isatty()
|
||||
self.convert = convert
|
||||
|
||||
# dict of ansi codes to win32 functions and parameters
|
||||
self.win32_calls = self.get_win32_calls()
|
||||
|
||||
# are we wrapping stderr?
|
||||
self.on_stderr = self.wrapped is sys.stderr
|
||||
|
||||
def should_wrap(self):
|
||||
'''
|
||||
True if this class is actually needed. If false, then the output
|
||||
stream will not be affected, nor will win32 calls be issued, so
|
||||
wrapping stdout is not actually required. This will generally be
|
||||
False on non-Windows platforms, unless optional functionality like
|
||||
autoreset has been requested using kwargs to init()
|
||||
'''
|
||||
return self.convert or self.strip or self.autoreset
|
||||
|
||||
def get_win32_calls(self):
|
||||
if self.convert and winterm:
|
||||
return {
|
||||
AnsiStyle.RESET_ALL: (winterm.reset_all, ),
|
||||
AnsiStyle.BRIGHT: (winterm.style, WinStyle.BRIGHT),
|
||||
AnsiStyle.DIM: (winterm.style, WinStyle.NORMAL),
|
||||
AnsiStyle.NORMAL: (winterm.style, WinStyle.NORMAL),
|
||||
AnsiFore.BLACK: (winterm.fore, WinColor.BLACK),
|
||||
AnsiFore.RED: (winterm.fore, WinColor.RED),
|
||||
AnsiFore.GREEN: (winterm.fore, WinColor.GREEN),
|
||||
AnsiFore.YELLOW: (winterm.fore, WinColor.YELLOW),
|
||||
AnsiFore.BLUE: (winterm.fore, WinColor.BLUE),
|
||||
AnsiFore.MAGENTA: (winterm.fore, WinColor.MAGENTA),
|
||||
AnsiFore.CYAN: (winterm.fore, WinColor.CYAN),
|
||||
AnsiFore.WHITE: (winterm.fore, WinColor.GREY),
|
||||
AnsiFore.RESET: (winterm.fore, ),
|
||||
AnsiFore.LIGHTBLACK_EX: (winterm.fore, WinColor.BLACK, True),
|
||||
AnsiFore.LIGHTRED_EX: (winterm.fore, WinColor.RED, True),
|
||||
AnsiFore.LIGHTGREEN_EX: (winterm.fore, WinColor.GREEN, True),
|
||||
AnsiFore.LIGHTYELLOW_EX: (winterm.fore, WinColor.YELLOW, True),
|
||||
AnsiFore.LIGHTBLUE_EX: (winterm.fore, WinColor.BLUE, True),
|
||||
AnsiFore.LIGHTMAGENTA_EX: (winterm.fore, WinColor.MAGENTA, True),
|
||||
AnsiFore.LIGHTCYAN_EX: (winterm.fore, WinColor.CYAN, True),
|
||||
AnsiFore.LIGHTWHITE_EX: (winterm.fore, WinColor.GREY, True),
|
||||
AnsiBack.BLACK: (winterm.back, WinColor.BLACK),
|
||||
AnsiBack.RED: (winterm.back, WinColor.RED),
|
||||
AnsiBack.GREEN: (winterm.back, WinColor.GREEN),
|
||||
AnsiBack.YELLOW: (winterm.back, WinColor.YELLOW),
|
||||
AnsiBack.BLUE: (winterm.back, WinColor.BLUE),
|
||||
AnsiBack.MAGENTA: (winterm.back, WinColor.MAGENTA),
|
||||
AnsiBack.CYAN: (winterm.back, WinColor.CYAN),
|
||||
AnsiBack.WHITE: (winterm.back, WinColor.GREY),
|
||||
AnsiBack.RESET: (winterm.back, ),
|
||||
AnsiBack.LIGHTBLACK_EX: (winterm.back, WinColor.BLACK, True),
|
||||
AnsiBack.LIGHTRED_EX: (winterm.back, WinColor.RED, True),
|
||||
AnsiBack.LIGHTGREEN_EX: (winterm.back, WinColor.GREEN, True),
|
||||
AnsiBack.LIGHTYELLOW_EX: (winterm.back, WinColor.YELLOW, True),
|
||||
AnsiBack.LIGHTBLUE_EX: (winterm.back, WinColor.BLUE, True),
|
||||
AnsiBack.LIGHTMAGENTA_EX: (winterm.back, WinColor.MAGENTA, True),
|
||||
AnsiBack.LIGHTCYAN_EX: (winterm.back, WinColor.CYAN, True),
|
||||
AnsiBack.LIGHTWHITE_EX: (winterm.back, WinColor.GREY, True),
|
||||
}
|
||||
return dict()
|
||||
|
||||
def write(self, text):
|
||||
if self.strip or self.convert:
|
||||
self.write_and_convert(text)
|
||||
else:
|
||||
self.wrapped.write(text)
|
||||
self.wrapped.flush()
|
||||
if self.autoreset:
|
||||
self.reset_all()
|
||||
|
||||
|
||||
def reset_all(self):
|
||||
if self.convert:
|
||||
self.call_win32('m', (0,))
|
||||
elif not self.strip and not self.stream.closed:
|
||||
self.wrapped.write(Style.RESET_ALL)
|
||||
|
||||
|
||||
def write_and_convert(self, text):
|
||||
'''
|
||||
Write the given text to our wrapped stream, stripping any ANSI
|
||||
sequences from the text, and optionally converting them into win32
|
||||
calls.
|
||||
'''
|
||||
cursor = 0
|
||||
text = self.convert_osc(text)
|
||||
for match in self.ANSI_CSI_RE.finditer(text):
|
||||
start, end = match.span()
|
||||
self.write_plain_text(text, cursor, start)
|
||||
self.convert_ansi(*match.groups())
|
||||
cursor = end
|
||||
self.write_plain_text(text, cursor, len(text))
|
||||
|
||||
|
||||
def write_plain_text(self, text, start, end):
|
||||
if start < end:
|
||||
self.wrapped.write(text[start:end])
|
||||
self.wrapped.flush()
|
||||
|
||||
|
||||
def convert_ansi(self, paramstring, command):
|
||||
if self.convert:
|
||||
params = self.extract_params(command, paramstring)
|
||||
self.call_win32(command, params)
|
||||
|
||||
|
||||
def extract_params(self, command, paramstring):
|
||||
if command in 'Hf':
|
||||
params = tuple(int(p) if len(p) != 0 else 1 for p in paramstring.split(';'))
|
||||
while len(params) < 2:
|
||||
# defaults:
|
||||
params = params + (1,)
|
||||
else:
|
||||
params = tuple(int(p) for p in paramstring.split(';') if len(p) != 0)
|
||||
if len(params) == 0:
|
||||
# defaults:
|
||||
if command in 'JKm':
|
||||
params = (0,)
|
||||
elif command in 'ABCD':
|
||||
params = (1,)
|
||||
|
||||
return params
|
||||
|
||||
|
||||
def call_win32(self, command, params):
|
||||
if command == 'm':
|
||||
for param in params:
|
||||
if param in self.win32_calls:
|
||||
func_args = self.win32_calls[param]
|
||||
func = func_args[0]
|
||||
args = func_args[1:]
|
||||
kwargs = dict(on_stderr=self.on_stderr)
|
||||
func(*args, **kwargs)
|
||||
elif command in 'J':
|
||||
winterm.erase_screen(params[0], on_stderr=self.on_stderr)
|
||||
elif command in 'K':
|
||||
winterm.erase_line(params[0], on_stderr=self.on_stderr)
|
||||
elif command in 'Hf': # cursor position - absolute
|
||||
winterm.set_cursor_position(params, on_stderr=self.on_stderr)
|
||||
elif command in 'ABCD': # cursor position - relative
|
||||
n = params[0]
|
||||
# A - up, B - down, C - forward, D - back
|
||||
x, y = {'A': (0, -n), 'B': (0, n), 'C': (n, 0), 'D': (-n, 0)}[command]
|
||||
winterm.cursor_adjust(x, y, on_stderr=self.on_stderr)
|
||||
|
||||
|
||||
def convert_osc(self, text):
|
||||
for match in self.ANSI_OSC_RE.finditer(text):
|
||||
start, end = match.span()
|
||||
text = text[:start] + text[end:]
|
||||
paramstring, command = match.groups()
|
||||
if command == BEL:
|
||||
if paramstring.count(";") == 1:
|
||||
params = paramstring.split(";")
|
||||
# 0 - change title and icon (we will only change title)
|
||||
# 1 - change icon (we don't support this)
|
||||
# 2 - change title
|
||||
if params[0] in '02':
|
||||
winterm.set_title(params[1])
|
||||
return text
|
||||
80
mosec/colorama/initialise.py
Normal file
80
mosec/colorama/initialise.py
Normal file
@@ -0,0 +1,80 @@
|
||||
# Copyright Jonathan Hartley 2013. BSD 3-Clause license, see LICENSE file.
|
||||
import atexit
|
||||
import contextlib
|
||||
import sys
|
||||
|
||||
from .ansitowin32 import AnsiToWin32
|
||||
|
||||
|
||||
orig_stdout = None
|
||||
orig_stderr = None
|
||||
|
||||
wrapped_stdout = None
|
||||
wrapped_stderr = None
|
||||
|
||||
atexit_done = False
|
||||
|
||||
|
||||
def reset_all():
|
||||
if AnsiToWin32 is not None: # Issue #74: objects might become None at exit
|
||||
AnsiToWin32(orig_stdout).reset_all()
|
||||
|
||||
|
||||
def init(autoreset=False, convert=None, strip=None, wrap=True):
|
||||
|
||||
if not wrap and any([autoreset, convert, strip]):
|
||||
raise ValueError('wrap=False conflicts with any other arg=True')
|
||||
|
||||
global wrapped_stdout, wrapped_stderr
|
||||
global orig_stdout, orig_stderr
|
||||
|
||||
orig_stdout = sys.stdout
|
||||
orig_stderr = sys.stderr
|
||||
|
||||
if sys.stdout is None:
|
||||
wrapped_stdout = None
|
||||
else:
|
||||
sys.stdout = wrapped_stdout = \
|
||||
wrap_stream(orig_stdout, convert, strip, autoreset, wrap)
|
||||
if sys.stderr is None:
|
||||
wrapped_stderr = None
|
||||
else:
|
||||
sys.stderr = wrapped_stderr = \
|
||||
wrap_stream(orig_stderr, convert, strip, autoreset, wrap)
|
||||
|
||||
global atexit_done
|
||||
if not atexit_done:
|
||||
atexit.register(reset_all)
|
||||
atexit_done = True
|
||||
|
||||
|
||||
def deinit():
|
||||
if orig_stdout is not None:
|
||||
sys.stdout = orig_stdout
|
||||
if orig_stderr is not None:
|
||||
sys.stderr = orig_stderr
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def colorama_text(*args, **kwargs):
|
||||
init(*args, **kwargs)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
deinit()
|
||||
|
||||
|
||||
def reinit():
|
||||
if wrapped_stdout is not None:
|
||||
sys.stdout = wrapped_stdout
|
||||
if wrapped_stderr is not None:
|
||||
sys.stderr = wrapped_stderr
|
||||
|
||||
|
||||
def wrap_stream(stream, convert, strip, autoreset, wrap):
|
||||
if wrap:
|
||||
wrapper = AnsiToWin32(stream,
|
||||
convert=convert, strip=strip, autoreset=autoreset)
|
||||
if wrapper.should_wrap():
|
||||
stream = wrapper.stream
|
||||
return stream
|
||||
152
mosec/colorama/win32.py
Normal file
152
mosec/colorama/win32.py
Normal file
@@ -0,0 +1,152 @@
|
||||
# Copyright Jonathan Hartley 2013. BSD 3-Clause license, see LICENSE file.
|
||||
|
||||
# from winbase.h
|
||||
STDOUT = -11
|
||||
STDERR = -12
|
||||
|
||||
try:
|
||||
import ctypes
|
||||
from ctypes import LibraryLoader
|
||||
windll = LibraryLoader(ctypes.WinDLL)
|
||||
from ctypes import wintypes
|
||||
except (AttributeError, ImportError):
|
||||
windll = None
|
||||
SetConsoleTextAttribute = lambda *_: None
|
||||
winapi_test = lambda *_: None
|
||||
else:
|
||||
from ctypes import byref, Structure, c_char, POINTER
|
||||
|
||||
COORD = wintypes._COORD
|
||||
|
||||
class CONSOLE_SCREEN_BUFFER_INFO(Structure):
|
||||
"""struct in wincon.h."""
|
||||
_fields_ = [
|
||||
("dwSize", COORD),
|
||||
("dwCursorPosition", COORD),
|
||||
("wAttributes", wintypes.WORD),
|
||||
("srWindow", wintypes.SMALL_RECT),
|
||||
("dwMaximumWindowSize", COORD),
|
||||
]
|
||||
def __str__(self):
|
||||
return '(%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d)' % (
|
||||
self.dwSize.Y, self.dwSize.X
|
||||
, self.dwCursorPosition.Y, self.dwCursorPosition.X
|
||||
, self.wAttributes
|
||||
, self.srWindow.Top, self.srWindow.Left, self.srWindow.Bottom, self.srWindow.Right
|
||||
, self.dwMaximumWindowSize.Y, self.dwMaximumWindowSize.X
|
||||
)
|
||||
|
||||
_GetStdHandle = windll.kernel32.GetStdHandle
|
||||
_GetStdHandle.argtypes = [
|
||||
wintypes.DWORD,
|
||||
]
|
||||
_GetStdHandle.restype = wintypes.HANDLE
|
||||
|
||||
_GetConsoleScreenBufferInfo = windll.kernel32.GetConsoleScreenBufferInfo
|
||||
_GetConsoleScreenBufferInfo.argtypes = [
|
||||
wintypes.HANDLE,
|
||||
POINTER(CONSOLE_SCREEN_BUFFER_INFO),
|
||||
]
|
||||
_GetConsoleScreenBufferInfo.restype = wintypes.BOOL
|
||||
|
||||
_SetConsoleTextAttribute = windll.kernel32.SetConsoleTextAttribute
|
||||
_SetConsoleTextAttribute.argtypes = [
|
||||
wintypes.HANDLE,
|
||||
wintypes.WORD,
|
||||
]
|
||||
_SetConsoleTextAttribute.restype = wintypes.BOOL
|
||||
|
||||
_SetConsoleCursorPosition = windll.kernel32.SetConsoleCursorPosition
|
||||
_SetConsoleCursorPosition.argtypes = [
|
||||
wintypes.HANDLE,
|
||||
COORD,
|
||||
]
|
||||
_SetConsoleCursorPosition.restype = wintypes.BOOL
|
||||
|
||||
_FillConsoleOutputCharacterA = windll.kernel32.FillConsoleOutputCharacterA
|
||||
_FillConsoleOutputCharacterA.argtypes = [
|
||||
wintypes.HANDLE,
|
||||
c_char,
|
||||
wintypes.DWORD,
|
||||
COORD,
|
||||
POINTER(wintypes.DWORD),
|
||||
]
|
||||
_FillConsoleOutputCharacterA.restype = wintypes.BOOL
|
||||
|
||||
_FillConsoleOutputAttribute = windll.kernel32.FillConsoleOutputAttribute
|
||||
_FillConsoleOutputAttribute.argtypes = [
|
||||
wintypes.HANDLE,
|
||||
wintypes.WORD,
|
||||
wintypes.DWORD,
|
||||
COORD,
|
||||
POINTER(wintypes.DWORD),
|
||||
]
|
||||
_FillConsoleOutputAttribute.restype = wintypes.BOOL
|
||||
|
||||
_SetConsoleTitleW = windll.kernel32.SetConsoleTitleW
|
||||
_SetConsoleTitleW.argtypes = [
|
||||
wintypes.LPCWSTR
|
||||
]
|
||||
_SetConsoleTitleW.restype = wintypes.BOOL
|
||||
|
||||
def _winapi_test(handle):
|
||||
csbi = CONSOLE_SCREEN_BUFFER_INFO()
|
||||
success = _GetConsoleScreenBufferInfo(
|
||||
handle, byref(csbi))
|
||||
return bool(success)
|
||||
|
||||
def winapi_test():
|
||||
return any(_winapi_test(h) for h in
|
||||
(_GetStdHandle(STDOUT), _GetStdHandle(STDERR)))
|
||||
|
||||
def GetConsoleScreenBufferInfo(stream_id=STDOUT):
|
||||
handle = _GetStdHandle(stream_id)
|
||||
csbi = CONSOLE_SCREEN_BUFFER_INFO()
|
||||
success = _GetConsoleScreenBufferInfo(
|
||||
handle, byref(csbi))
|
||||
return csbi
|
||||
|
||||
def SetConsoleTextAttribute(stream_id, attrs):
|
||||
handle = _GetStdHandle(stream_id)
|
||||
return _SetConsoleTextAttribute(handle, attrs)
|
||||
|
||||
def SetConsoleCursorPosition(stream_id, position, adjust=True):
|
||||
position = COORD(*position)
|
||||
# If the position is out of range, do nothing.
|
||||
if position.Y <= 0 or position.X <= 0:
|
||||
return
|
||||
# Adjust for Windows' SetConsoleCursorPosition:
|
||||
# 1. being 0-based, while ANSI is 1-based.
|
||||
# 2. expecting (x,y), while ANSI uses (y,x).
|
||||
adjusted_position = COORD(position.Y - 1, position.X - 1)
|
||||
if adjust:
|
||||
# Adjust for viewport's scroll position
|
||||
sr = GetConsoleScreenBufferInfo(STDOUT).srWindow
|
||||
adjusted_position.Y += sr.Top
|
||||
adjusted_position.X += sr.Left
|
||||
# Resume normal processing
|
||||
handle = _GetStdHandle(stream_id)
|
||||
return _SetConsoleCursorPosition(handle, adjusted_position)
|
||||
|
||||
def FillConsoleOutputCharacter(stream_id, char, length, start):
|
||||
handle = _GetStdHandle(stream_id)
|
||||
char = c_char(char.encode())
|
||||
length = wintypes.DWORD(length)
|
||||
num_written = wintypes.DWORD(0)
|
||||
# Note that this is hard-coded for ANSI (vs wide) bytes.
|
||||
success = _FillConsoleOutputCharacterA(
|
||||
handle, char, length, start, byref(num_written))
|
||||
return num_written.value
|
||||
|
||||
def FillConsoleOutputAttribute(stream_id, attr, length, start):
|
||||
''' FillConsoleOutputAttribute( hConsole, csbi.wAttributes, dwConSize, coordScreen, &cCharsWritten )'''
|
||||
handle = _GetStdHandle(stream_id)
|
||||
attribute = wintypes.WORD(attr)
|
||||
length = wintypes.DWORD(length)
|
||||
num_written = wintypes.DWORD(0)
|
||||
# Note that this is hard-coded for ANSI (vs wide) bytes.
|
||||
return _FillConsoleOutputAttribute(
|
||||
handle, attribute, length, start, byref(num_written))
|
||||
|
||||
def SetConsoleTitle(title):
|
||||
return _SetConsoleTitleW(title)
|
||||
169
mosec/colorama/winterm.py
Normal file
169
mosec/colorama/winterm.py
Normal file
@@ -0,0 +1,169 @@
|
||||
# Copyright Jonathan Hartley 2013. BSD 3-Clause license, see LICENSE file.
|
||||
from . import win32
|
||||
|
||||
|
||||
# from wincon.h
|
||||
class WinColor(object):
|
||||
BLACK = 0
|
||||
BLUE = 1
|
||||
GREEN = 2
|
||||
CYAN = 3
|
||||
RED = 4
|
||||
MAGENTA = 5
|
||||
YELLOW = 6
|
||||
GREY = 7
|
||||
|
||||
# from wincon.h
|
||||
class WinStyle(object):
|
||||
NORMAL = 0x00 # dim text, dim background
|
||||
BRIGHT = 0x08 # bright text, dim background
|
||||
BRIGHT_BACKGROUND = 0x80 # dim text, bright background
|
||||
|
||||
class WinTerm(object):
|
||||
|
||||
def __init__(self):
|
||||
self._default = win32.GetConsoleScreenBufferInfo(win32.STDOUT).wAttributes
|
||||
self.set_attrs(self._default)
|
||||
self._default_fore = self._fore
|
||||
self._default_back = self._back
|
||||
self._default_style = self._style
|
||||
# In order to emulate LIGHT_EX in windows, we borrow the BRIGHT style.
|
||||
# So that LIGHT_EX colors and BRIGHT style do not clobber each other,
|
||||
# we track them separately, since LIGHT_EX is overwritten by Fore/Back
|
||||
# and BRIGHT is overwritten by Style codes.
|
||||
self._light = 0
|
||||
|
||||
def get_attrs(self):
|
||||
return self._fore + self._back * 16 + (self._style | self._light)
|
||||
|
||||
def set_attrs(self, value):
|
||||
self._fore = value & 7
|
||||
self._back = (value >> 4) & 7
|
||||
self._style = value & (WinStyle.BRIGHT | WinStyle.BRIGHT_BACKGROUND)
|
||||
|
||||
def reset_all(self, on_stderr=None):
|
||||
self.set_attrs(self._default)
|
||||
self.set_console(attrs=self._default)
|
||||
self._light = 0
|
||||
|
||||
def fore(self, fore=None, light=False, on_stderr=False):
|
||||
if fore is None:
|
||||
fore = self._default_fore
|
||||
self._fore = fore
|
||||
# Emulate LIGHT_EX with BRIGHT Style
|
||||
if light:
|
||||
self._light |= WinStyle.BRIGHT
|
||||
else:
|
||||
self._light &= ~WinStyle.BRIGHT
|
||||
self.set_console(on_stderr=on_stderr)
|
||||
|
||||
def back(self, back=None, light=False, on_stderr=False):
|
||||
if back is None:
|
||||
back = self._default_back
|
||||
self._back = back
|
||||
# Emulate LIGHT_EX with BRIGHT_BACKGROUND Style
|
||||
if light:
|
||||
self._light |= WinStyle.BRIGHT_BACKGROUND
|
||||
else:
|
||||
self._light &= ~WinStyle.BRIGHT_BACKGROUND
|
||||
self.set_console(on_stderr=on_stderr)
|
||||
|
||||
def style(self, style=None, on_stderr=False):
|
||||
if style is None:
|
||||
style = self._default_style
|
||||
self._style = style
|
||||
self.set_console(on_stderr=on_stderr)
|
||||
|
||||
def set_console(self, attrs=None, on_stderr=False):
|
||||
if attrs is None:
|
||||
attrs = self.get_attrs()
|
||||
handle = win32.STDOUT
|
||||
if on_stderr:
|
||||
handle = win32.STDERR
|
||||
win32.SetConsoleTextAttribute(handle, attrs)
|
||||
|
||||
def get_position(self, handle):
|
||||
position = win32.GetConsoleScreenBufferInfo(handle).dwCursorPosition
|
||||
# Because Windows coordinates are 0-based,
|
||||
# and win32.SetConsoleCursorPosition expects 1-based.
|
||||
position.X += 1
|
||||
position.Y += 1
|
||||
return position
|
||||
|
||||
def set_cursor_position(self, position=None, on_stderr=False):
|
||||
if position is None:
|
||||
# I'm not currently tracking the position, so there is no default.
|
||||
# position = self.get_position()
|
||||
return
|
||||
handle = win32.STDOUT
|
||||
if on_stderr:
|
||||
handle = win32.STDERR
|
||||
win32.SetConsoleCursorPosition(handle, position)
|
||||
|
||||
def cursor_adjust(self, x, y, on_stderr=False):
|
||||
handle = win32.STDOUT
|
||||
if on_stderr:
|
||||
handle = win32.STDERR
|
||||
position = self.get_position(handle)
|
||||
adjusted_position = (position.Y + y, position.X + x)
|
||||
win32.SetConsoleCursorPosition(handle, adjusted_position, adjust=False)
|
||||
|
||||
def erase_screen(self, mode=0, on_stderr=False):
|
||||
# 0 should clear from the cursor to the end of the screen.
|
||||
# 1 should clear from the cursor to the beginning of the screen.
|
||||
# 2 should clear the entire screen, and move cursor to (1,1)
|
||||
handle = win32.STDOUT
|
||||
if on_stderr:
|
||||
handle = win32.STDERR
|
||||
csbi = win32.GetConsoleScreenBufferInfo(handle)
|
||||
# get the number of character cells in the current buffer
|
||||
cells_in_screen = csbi.dwSize.X * csbi.dwSize.Y
|
||||
# get number of character cells before current cursor position
|
||||
cells_before_cursor = csbi.dwSize.X * csbi.dwCursorPosition.Y + csbi.dwCursorPosition.X
|
||||
if mode == 0:
|
||||
from_coord = csbi.dwCursorPosition
|
||||
cells_to_erase = cells_in_screen - cells_before_cursor
|
||||
elif mode == 1:
|
||||
from_coord = win32.COORD(0, 0)
|
||||
cells_to_erase = cells_before_cursor
|
||||
elif mode == 2:
|
||||
from_coord = win32.COORD(0, 0)
|
||||
cells_to_erase = cells_in_screen
|
||||
else:
|
||||
# invalid mode
|
||||
return
|
||||
# fill the entire screen with blanks
|
||||
win32.FillConsoleOutputCharacter(handle, ' ', cells_to_erase, from_coord)
|
||||
# now set the buffer's attributes accordingly
|
||||
win32.FillConsoleOutputAttribute(handle, self.get_attrs(), cells_to_erase, from_coord)
|
||||
if mode == 2:
|
||||
# put the cursor where needed
|
||||
win32.SetConsoleCursorPosition(handle, (1, 1))
|
||||
|
||||
def erase_line(self, mode=0, on_stderr=False):
|
||||
# 0 should clear from the cursor to the end of the line.
|
||||
# 1 should clear from the cursor to the beginning of the line.
|
||||
# 2 should clear the entire line.
|
||||
handle = win32.STDOUT
|
||||
if on_stderr:
|
||||
handle = win32.STDERR
|
||||
csbi = win32.GetConsoleScreenBufferInfo(handle)
|
||||
if mode == 0:
|
||||
from_coord = csbi.dwCursorPosition
|
||||
cells_to_erase = csbi.dwSize.X - csbi.dwCursorPosition.X
|
||||
elif mode == 1:
|
||||
from_coord = win32.COORD(0, csbi.dwCursorPosition.Y)
|
||||
cells_to_erase = csbi.dwCursorPosition.X
|
||||
elif mode == 2:
|
||||
from_coord = win32.COORD(0, csbi.dwCursorPosition.Y)
|
||||
cells_to_erase = csbi.dwSize.X
|
||||
else:
|
||||
# invalid mode
|
||||
return
|
||||
# fill the entire screen with blanks
|
||||
win32.FillConsoleOutputCharacter(handle, ' ', cells_to_erase, from_coord)
|
||||
# now set the buffer's attributes accordingly
|
||||
win32.FillConsoleOutputAttribute(handle, self.get_attrs(), cells_to_erase, from_coord)
|
||||
|
||||
def set_title(self, title):
|
||||
win32.SetConsoleTitle(title)
|
||||
30
mosec/mosec_log_helper.py
Normal file
30
mosec/mosec_log_helper.py
Normal file
@@ -0,0 +1,30 @@
|
||||
import logging
|
||||
import sys
|
||||
from mosec.colorama.ansi import Fore, Style
|
||||
|
||||
|
||||
class Logger(object):
|
||||
def __init__(self, name, level=logging.INFO):
|
||||
self.logger = logging.getLogger(name=name)
|
||||
# set lowest info critical > error > warning > info > debug
|
||||
self.logger.setLevel(level)
|
||||
|
||||
self.ch = logging.StreamHandler(sys.stdout)
|
||||
self.ch.setLevel(level)
|
||||
self.logger.addHandler(self.ch)
|
||||
|
||||
def debug(self, msg):
|
||||
self.logger.debug(str(msg))
|
||||
|
||||
def warn(self, msg):
|
||||
self.logger.warning(Fore.YELLOW + str(msg) + Style.RESET_ALL)
|
||||
|
||||
def info(self, msg):
|
||||
self.logger.info(Fore.LIGHTGREEN_EX + str(msg) + Style.RESET_ALL)
|
||||
|
||||
def error(self, msg):
|
||||
self.logger.error(Fore.LIGHTRED_EX + str(msg) + Style.RESET_ALL)
|
||||
|
||||
def set_log_level(self, level):
|
||||
self.logger.setLevel(level)
|
||||
self.ch.setLevel(level)
|
||||
265
mosec/pip_resolve.py
Normal file
265
mosec/pip_resolve.py
Normal file
@@ -0,0 +1,265 @@
|
||||
import logging
|
||||
import sys
|
||||
import os
|
||||
import argparse
|
||||
import json
|
||||
import ssl
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from operator import attrgetter
|
||||
from mosec import mosec_log_helper
|
||||
from mosec import setup_file
|
||||
from mosec import utils
|
||||
from mosec.requirement_file_parser import get_requirements_list
|
||||
from mosec.requirement_dist import ReqDist
|
||||
|
||||
try:
|
||||
import pkg_resources
|
||||
except ImportError:
|
||||
# try using the version vendored by pip
|
||||
try:
|
||||
import pip._vendor.pkg_resources as pkg_resources
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"Could not import pkg_resources; please install setuptools or pip.")
|
||||
|
||||
try:
|
||||
from collections import OrderedDict
|
||||
except ImportError:
|
||||
from ordereddict import OrderedDict
|
||||
|
||||
|
||||
log = mosec_log_helper.Logger(name="mosec")
|
||||
|
||||
|
||||
def create_deps_tree(
|
||||
dist_tree,
|
||||
top_level_requirements,
|
||||
req_file_path,
|
||||
allow_missing=False,
|
||||
only_provenance=False
|
||||
):
|
||||
"""Create dist dependencies tree
|
||||
:param dict dist_tree: the installed dists tree
|
||||
:param list top_level_requirements: list of required dists
|
||||
:param str req_file_path: path to the dependencies file (e.g. requirements.txt)
|
||||
:param bool allow_missing: ignore uninstalled dependencies
|
||||
:param bool only_provenance: only care provenance dependencies
|
||||
:rtype: dict
|
||||
"""
|
||||
DEPENDENCIES = 'dependencies'
|
||||
VERSION = 'version'
|
||||
NAME = 'name'
|
||||
DIR_VERSION = '1.0.0'
|
||||
FROM = 'from'
|
||||
|
||||
tree = OrderedDict(
|
||||
sorted(
|
||||
[(k, sorted(v, key=attrgetter('key'))) for k, v in dist_tree.items()]
|
||||
, key=lambda kv: kv[0].key
|
||||
)
|
||||
)
|
||||
nodes = tree.keys()
|
||||
key_tree = dict((k.key, v) for k, v in tree.items())
|
||||
|
||||
top_level_req_lower_names = [p.name.lower() for p in top_level_requirements]
|
||||
top_level_req_dists = [p for p in nodes if p.key in top_level_req_lower_names]
|
||||
|
||||
def _create_children_recursive(root_package, ancestors):
|
||||
root_name = root_package[NAME]
|
||||
if root_name.lower() not in key_tree:
|
||||
msg = 'Required packages missing: ' + root_name
|
||||
if allow_missing:
|
||||
log.error(msg)
|
||||
return
|
||||
else:
|
||||
sys.exit(msg)
|
||||
|
||||
ancestors = ancestors.copy()
|
||||
ancestors.add(root_name.lower())
|
||||
children_dists = key_tree[root_name.lower()]
|
||||
for child_dist in children_dists:
|
||||
if child_dist.key in ancestors:
|
||||
continue
|
||||
|
||||
child_node = _create_tree_node(child_dist, root_package)
|
||||
_create_children_recursive(child_node, ancestors)
|
||||
root_package[DEPENDENCIES][child_node[NAME]] = child_node
|
||||
return root_package
|
||||
|
||||
def _create_root():
|
||||
name, version = None, None
|
||||
if os.path.basename(req_file_path) == 'setup.py':
|
||||
with open(req_file_path, "r") as setup_py_file:
|
||||
name, version = setup_file.parse_name_and_version(setup_py_file.read())
|
||||
|
||||
root = {
|
||||
NAME: name or os.path.basename(os.path.dirname(os.path.abspath(req_file_path))),
|
||||
VERSION: version or DIR_VERSION,
|
||||
DEPENDENCIES: {}
|
||||
}
|
||||
root[FROM] = [root[NAME] + '@' + root[VERSION]]
|
||||
return root
|
||||
|
||||
def _create_tree_node(dist_node, parent):
|
||||
version = dist_node.version
|
||||
if isinstance(version, tuple):
|
||||
version = '.'.join(map(str, version))
|
||||
return {
|
||||
NAME: dist_node.project_name,
|
||||
VERSION: version,
|
||||
FROM: parent[FROM] + [dist_node.project_name + '@' + version],
|
||||
DEPENDENCIES: {}
|
||||
}
|
||||
|
||||
tree_root = _create_root()
|
||||
for dist in top_level_req_dists:
|
||||
tree_node = _create_tree_node(dist, tree_root)
|
||||
if only_provenance:
|
||||
tree_root[DEPENDENCIES][tree_node[NAME]] = tree_node
|
||||
else:
|
||||
tree_root[DEPENDENCIES][tree_node[NAME]] = _create_children_recursive(tree_node, set([]))
|
||||
return tree_root
|
||||
|
||||
|
||||
def create_dependencies_tree_by_req_file(
|
||||
requirements_file,
|
||||
allow_missing=False,
|
||||
only_provenance=False
|
||||
):
|
||||
"""Create dist dependencies tree from file
|
||||
:param str requirements_file: path to the dependencies file (e.g. requirements.txt)
|
||||
:param bool allow_missing: ignore uninstalled dependencies
|
||||
:param bool only_provenance: only care provenance dependencies
|
||||
:rtype: dict
|
||||
"""
|
||||
|
||||
# get all installed package distribution object list
|
||||
dists = list(pkg_resources.working_set)
|
||||
dists_dict = dict((p.key, p) for p in dists)
|
||||
dists_tree = dict((p, [ReqDist(r, dists_dict.get(r.key)) for r in p.requires()]) for p in dists_dict.values())
|
||||
|
||||
required = get_requirements_list(requirements_file)
|
||||
installed = [utils.canonicalize_dist_name(d) for d in dists_dict]
|
||||
top_level_requirements = []
|
||||
missing_package_names = []
|
||||
for r in required:
|
||||
if utils.canonicalize_dist_name(r.name) not in installed:
|
||||
missing_package_names.append(r.name)
|
||||
else:
|
||||
top_level_requirements.append(r)
|
||||
if missing_package_names:
|
||||
msg = 'Required packages missing: ' + (', '.join(missing_package_names))
|
||||
if allow_missing:
|
||||
log.error(msg)
|
||||
else:
|
||||
sys.exit(msg)
|
||||
|
||||
return create_deps_tree(
|
||||
dists_tree, top_level_requirements, requirements_file, allow_missing, only_provenance)
|
||||
|
||||
|
||||
def render_response(response_json):
|
||||
|
||||
def _print_single_vuln(vuln):
|
||||
log.error("✗ {} severity vulnerability ({} - {}) found on {}@{}".format(
|
||||
vuln.get('severity'),
|
||||
vuln.get('title', ''),
|
||||
vuln.get('cve', ''),
|
||||
vuln.get('packageName'),
|
||||
vuln.get('version'))
|
||||
)
|
||||
if vuln.get('from', None):
|
||||
from_arr = vuln.get('from')
|
||||
from_str = ""
|
||||
for _from in from_arr:
|
||||
from_str += _from + " > "
|
||||
from_str = from_str[:-3]
|
||||
print("- from: {}".format(from_str))
|
||||
if vuln.get('target_version', []):
|
||||
log.info("! Fix version {}".format(vuln.get('target_version')))
|
||||
print("")
|
||||
|
||||
if response_json.get('ok', False):
|
||||
log.info("✓ Tested {} dependencies for known vulnerabilities, no vulnerable paths found."
|
||||
.format(response_json.get('dependencyCount', 0)))
|
||||
elif response_json.get('vulnerabilities', None):
|
||||
vulns = response_json.get('vulnerabilities')
|
||||
for vuln in vulns:
|
||||
_print_single_vuln(vuln)
|
||||
|
||||
log.warn("Tested {} dependencies for known vulnerabilities, found {} vulnerable paths."
|
||||
.format(response_json.get('dependencyCount', 0), len(vulns)))
|
||||
|
||||
|
||||
def run(args):
|
||||
deps_tree = create_dependencies_tree_by_req_file(
|
||||
args.requirements,
|
||||
allow_missing=args.allow_missing,
|
||||
only_provenance=args.only_provenance,
|
||||
)
|
||||
|
||||
deps_tree['severityLevel'] = args.level
|
||||
deps_tree['type'] = 'pip'
|
||||
deps_tree['language'] = 'python'
|
||||
|
||||
log.debug(json.dumps(deps_tree, indent=2))
|
||||
|
||||
ctx = ssl.create_default_context()
|
||||
ctx.check_hostname = False
|
||||
ctx.verify_mode = ssl.CERT_NONE
|
||||
|
||||
req = urllib.request.Request(
|
||||
method='POST',
|
||||
url=args.endpoint,
|
||||
headers={
|
||||
'Content-Type': 'application/json'
|
||||
},
|
||||
data=bytes(json.dumps(deps_tree).encode('utf-8'))
|
||||
)
|
||||
try:
|
||||
response = urllib.request.urlopen(req, timeout=15, context=ctx)
|
||||
response_json = json.loads(response.read().decode('utf-8'))
|
||||
render_response(response_json)
|
||||
|
||||
if not response_json.get('ok', False):
|
||||
return 1
|
||||
except urllib.error.HTTPError as e:
|
||||
raise Exception("Network Error: {}".format(e))
|
||||
except json.JSONDecodeError as e:
|
||||
raise Exception("API return data format error.")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("requirements",
|
||||
help="依赖文件 (requirements.txt 或 Pipfile)")
|
||||
parser.add_argument("--endpoint",
|
||||
action="store",
|
||||
required=True,
|
||||
help="上报API")
|
||||
parser.add_argument("--allow-missing",
|
||||
action="store_true",
|
||||
help="忽略未安装的依赖")
|
||||
parser.add_argument("--only-provenance",
|
||||
action="store_true",
|
||||
help="仅检查直接依赖")
|
||||
parser.add_argument("--level",
|
||||
action="store",
|
||||
default="High",
|
||||
help="威胁等级 [High|Medium|Low]. default: High")
|
||||
parser.add_argument("--no-except",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="发现漏洞不抛出异常")
|
||||
parser.add_argument("--debug",
|
||||
action="store_true",
|
||||
default=False)
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.debug:
|
||||
log.set_log_level(logging.DEBUG)
|
||||
|
||||
status = run(args)
|
||||
if status == 1 and not args.no_except:
|
||||
raise BaseException("Found Vulnerable!")
|
||||
63
mosec/pipfile.py
Normal file
63
mosec/pipfile.py
Normal file
@@ -0,0 +1,63 @@
|
||||
"""Simplistic parsing of Pipfile dependency files
|
||||
|
||||
This only extracts a small subset of the information present in a Pipfile,
|
||||
as needed for the purposes of this library.
|
||||
"""
|
||||
from .utils import is_string
|
||||
from .pytoml import loads as pytoml_loads
|
||||
|
||||
|
||||
class PipfileRequirement(object):
|
||||
def __init__(self, name):
|
||||
self.name = name
|
||||
|
||||
self.editable = False
|
||||
self.vcs = None
|
||||
self.vcs_uri = None
|
||||
self.version = None
|
||||
self.markers = None
|
||||
self.provenance = None # a tuple of (file name, line)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, name, requirement_dict, pos_in_toml):
|
||||
req = cls(name)
|
||||
|
||||
req.version = requirement_dict.get('version')
|
||||
req.editable = requirement_dict.get('editable', False)
|
||||
for vcs in ['git', 'hg', 'svn', 'bzr']:
|
||||
if vcs in requirement_dict:
|
||||
req.vcs = vcs
|
||||
req.vcs_uri = requirement_dict[vcs]
|
||||
break
|
||||
req.markers = requirement_dict.get('markers')
|
||||
# proper file name to be injected into provenance by the calling code
|
||||
req.provenance = ('Pipfile', pos_in_toml[0], pos_in_toml[0])
|
||||
|
||||
return req
|
||||
|
||||
|
||||
def val_with_pos(kind, text, value, pos):
|
||||
return (value, pos)
|
||||
|
||||
|
||||
def parse(file_contents):
|
||||
data = pytoml_loads(file_contents, translate=val_with_pos)
|
||||
|
||||
sections = ['packages', 'dev-packages']
|
||||
res = dict.fromkeys(sections)
|
||||
for section in sections:
|
||||
if section not in data:
|
||||
continue
|
||||
|
||||
section_data = data[section]
|
||||
|
||||
res[section] = [
|
||||
PipfileRequirement.from_dict(
|
||||
name,
|
||||
value if not is_string(value) else {'version': value},
|
||||
pos,
|
||||
)
|
||||
for name, (value, pos) in sorted(section_data.items())
|
||||
]
|
||||
|
||||
return res
|
||||
16
mosec/pytoml/LICENSE
Normal file
16
mosec/pytoml/LICENSE
Normal file
@@ -0,0 +1,16 @@
|
||||
No-notice MIT License
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
THE SOFTWARE.
|
||||
9
mosec/pytoml/README.txt
Normal file
9
mosec/pytoml/README.txt
Normal file
@@ -0,0 +1,9 @@
|
||||
This is pytoml v0.1.16, taken from the avakar/pytoml GitHub repo.
|
||||
|
||||
See: https://github.com/avakar/pytoml/releases/tag/v0.1.16
|
||||
|
||||
It is bundled out of necessity due to constraints of the
|
||||
Snyk CLI plugin architecture.
|
||||
|
||||
Also, it contains a Snyk-specific patch: the user-configurable "translate"
|
||||
function now also takes a fourth argument, "pos".
|
||||
3
mosec/pytoml/__init__.py
Normal file
3
mosec/pytoml/__init__.py
Normal file
@@ -0,0 +1,3 @@
|
||||
from .core import TomlError
|
||||
from .parser import load, loads
|
||||
from .writer import dump, dumps
|
||||
13
mosec/pytoml/core.py
Normal file
13
mosec/pytoml/core.py
Normal file
@@ -0,0 +1,13 @@
|
||||
class TomlError(RuntimeError):
|
||||
def __init__(self, message, line, col, filename):
|
||||
RuntimeError.__init__(self, message, line, col, filename)
|
||||
self.message = message
|
||||
self.line = line
|
||||
self.col = col
|
||||
self.filename = filename
|
||||
|
||||
def __str__(self):
|
||||
return '{}({}, {}): {}'.format(self.filename, self.line, self.col, self.message)
|
||||
|
||||
def __repr__(self):
|
||||
return 'TomlError({!r}, {!r}, {!r}, {!r})'.format(self.message, self.line, self.col, self.filename)
|
||||
374
mosec/pytoml/parser.py
Normal file
374
mosec/pytoml/parser.py
Normal file
@@ -0,0 +1,374 @@
|
||||
import string, re, sys, datetime
|
||||
from .core import TomlError
|
||||
|
||||
if sys.version_info[0] == 2:
|
||||
_chr = unichr
|
||||
else:
|
||||
_chr = chr
|
||||
|
||||
def load(fin, translate=lambda t, x, v, pos: v, object_pairs_hook=dict):
|
||||
return loads(fin.read(), translate=translate, object_pairs_hook=object_pairs_hook, filename=getattr(fin, 'name', repr(fin)))
|
||||
|
||||
def loads(s, filename='<string>', translate=lambda t, x, v, pos: v, object_pairs_hook=dict):
|
||||
if isinstance(s, bytes):
|
||||
s = s.decode('utf-8')
|
||||
|
||||
s = s.replace('\r\n', '\n')
|
||||
|
||||
root = object_pairs_hook()
|
||||
tables = object_pairs_hook()
|
||||
scope = root
|
||||
|
||||
src = _Source(s, filename=filename)
|
||||
ast = _p_toml(src, object_pairs_hook=object_pairs_hook)
|
||||
|
||||
def error(msg):
|
||||
raise TomlError(msg, pos[0], pos[1], filename)
|
||||
|
||||
def process_value(v, object_pairs_hook):
|
||||
kind, text, value, pos = v
|
||||
if kind == 'str' and value.startswith('\n'):
|
||||
value = value[1:]
|
||||
if kind == 'array':
|
||||
if value and any(k != value[0][0] for k, t, v, p in value[1:]):
|
||||
error('array-type-mismatch')
|
||||
value = [process_value(item, object_pairs_hook=object_pairs_hook) for item in value]
|
||||
elif kind == 'table':
|
||||
value = object_pairs_hook([(k, process_value(value[k], object_pairs_hook=object_pairs_hook)) for k in value])
|
||||
return translate(kind, text, value, pos)
|
||||
|
||||
for kind, value, pos in ast:
|
||||
if kind == 'kv':
|
||||
k, v = value
|
||||
if k in scope:
|
||||
error('duplicate_keys. Key "{0}" was used more than once.'.format(k))
|
||||
scope[k] = process_value(v, object_pairs_hook=object_pairs_hook)
|
||||
else:
|
||||
is_table_array = (kind == 'table_array')
|
||||
cur = tables
|
||||
for name in value[:-1]:
|
||||
if isinstance(cur.get(name), list):
|
||||
d, cur = cur[name][-1]
|
||||
else:
|
||||
d, cur = cur.setdefault(name, (None, object_pairs_hook()))
|
||||
|
||||
scope = object_pairs_hook()
|
||||
name = value[-1]
|
||||
if name not in cur:
|
||||
if is_table_array:
|
||||
cur[name] = [(scope, object_pairs_hook())]
|
||||
else:
|
||||
cur[name] = (scope, object_pairs_hook())
|
||||
elif isinstance(cur[name], list):
|
||||
if not is_table_array:
|
||||
error('table_type_mismatch')
|
||||
cur[name].append((scope, object_pairs_hook()))
|
||||
else:
|
||||
if is_table_array:
|
||||
error('table_type_mismatch')
|
||||
old_scope, next_table = cur[name]
|
||||
if old_scope is not None:
|
||||
error('duplicate_tables')
|
||||
cur[name] = (scope, next_table)
|
||||
|
||||
def merge_tables(scope, tables):
|
||||
if scope is None:
|
||||
scope = object_pairs_hook()
|
||||
for k in tables:
|
||||
if k in scope:
|
||||
error('key_table_conflict')
|
||||
v = tables[k]
|
||||
if isinstance(v, list):
|
||||
scope[k] = [merge_tables(sc, tbl) for sc, tbl in v]
|
||||
else:
|
||||
scope[k] = merge_tables(v[0], v[1])
|
||||
return scope
|
||||
|
||||
return merge_tables(root, tables)
|
||||
|
||||
class _Source:
|
||||
def __init__(self, s, filename=None):
|
||||
self.s = s
|
||||
self._pos = (1, 1)
|
||||
self._last = None
|
||||
self._filename = filename
|
||||
self.backtrack_stack = []
|
||||
|
||||
def last(self):
|
||||
return self._last
|
||||
|
||||
def pos(self):
|
||||
return self._pos
|
||||
|
||||
def fail(self):
|
||||
return self._expect(None)
|
||||
|
||||
def consume_dot(self):
|
||||
if self.s:
|
||||
self._last = self.s[0]
|
||||
self.s = self[1:]
|
||||
self._advance(self._last)
|
||||
return self._last
|
||||
return None
|
||||
|
||||
def expect_dot(self):
|
||||
return self._expect(self.consume_dot())
|
||||
|
||||
def consume_eof(self):
|
||||
if not self.s:
|
||||
self._last = ''
|
||||
return True
|
||||
return False
|
||||
|
||||
def expect_eof(self):
|
||||
return self._expect(self.consume_eof())
|
||||
|
||||
def consume(self, s):
|
||||
if self.s.startswith(s):
|
||||
self.s = self.s[len(s):]
|
||||
self._last = s
|
||||
self._advance(s)
|
||||
return True
|
||||
return False
|
||||
|
||||
def expect(self, s):
|
||||
return self._expect(self.consume(s))
|
||||
|
||||
def consume_re(self, re):
|
||||
m = re.match(self.s)
|
||||
if m:
|
||||
self.s = self.s[len(m.group(0)):]
|
||||
self._last = m
|
||||
self._advance(m.group(0))
|
||||
return m
|
||||
return None
|
||||
|
||||
def expect_re(self, re):
|
||||
return self._expect(self.consume_re(re))
|
||||
|
||||
def __enter__(self):
|
||||
self.backtrack_stack.append((self.s, self._pos))
|
||||
|
||||
def __exit__(self, type, value, traceback):
|
||||
if type is None:
|
||||
self.backtrack_stack.pop()
|
||||
else:
|
||||
self.s, self._pos = self.backtrack_stack.pop()
|
||||
return type == TomlError
|
||||
|
||||
def commit(self):
|
||||
self.backtrack_stack[-1] = (self.s, self._pos)
|
||||
|
||||
def _expect(self, r):
|
||||
if not r:
|
||||
raise TomlError('msg', self._pos[0], self._pos[1], self._filename)
|
||||
return r
|
||||
|
||||
def _advance(self, s):
|
||||
suffix_pos = s.rfind('\n')
|
||||
if suffix_pos == -1:
|
||||
self._pos = (self._pos[0], self._pos[1] + len(s))
|
||||
else:
|
||||
self._pos = (self._pos[0] + s.count('\n'), len(s) - suffix_pos)
|
||||
|
||||
_ews_re = re.compile(r'(?:[ \t]|#[^\n]*\n|#[^\n]*\Z|\n)*')
|
||||
def _p_ews(s):
|
||||
s.expect_re(_ews_re)
|
||||
|
||||
_ws_re = re.compile(r'[ \t]*')
|
||||
def _p_ws(s):
|
||||
s.expect_re(_ws_re)
|
||||
|
||||
_escapes = { 'b': '\b', 'n': '\n', 'r': '\r', 't': '\t', '"': '"', '\'': '\'',
|
||||
'\\': '\\', '/': '/', 'f': '\f' }
|
||||
|
||||
_basicstr_re = re.compile(r'[^"\\\000-\037]*')
|
||||
_short_uni_re = re.compile(r'u([0-9a-fA-F]{4})')
|
||||
_long_uni_re = re.compile(r'U([0-9a-fA-F]{8})')
|
||||
_escapes_re = re.compile('[bnrt"\'\\\\/f]')
|
||||
_newline_esc_re = re.compile('\n[ \t\n]*')
|
||||
def _p_basicstr_content(s, content=_basicstr_re):
|
||||
res = []
|
||||
while True:
|
||||
res.append(s.expect_re(content).group(0))
|
||||
if not s.consume('\\'):
|
||||
break
|
||||
if s.consume_re(_newline_esc_re):
|
||||
pass
|
||||
elif s.consume_re(_short_uni_re) or s.consume_re(_long_uni_re):
|
||||
res.append(_chr(int(s.last().group(1), 16)))
|
||||
else:
|
||||
s.expect_re(_escapes_re)
|
||||
res.append(_escapes[s.last().group(0)])
|
||||
return ''.join(res)
|
||||
|
||||
_key_re = re.compile(r'[0-9a-zA-Z-_]+')
|
||||
def _p_key(s):
|
||||
with s:
|
||||
s.expect('"')
|
||||
r = _p_basicstr_content(s, _basicstr_re)
|
||||
s.expect('"')
|
||||
return r
|
||||
if s.consume('\''):
|
||||
if s.consume('\'\''):
|
||||
r = s.expect_re(_litstr_ml_re).group(0)
|
||||
s.expect('\'\'\'')
|
||||
else:
|
||||
r = s.expect_re(_litstr_re).group(0)
|
||||
s.expect('\'')
|
||||
return r
|
||||
return s.expect_re(_key_re).group(0)
|
||||
|
||||
_float_re = re.compile(r'[+-]?(?:0|[1-9](?:_?\d)*)(?:\.\d(?:_?\d)*)?(?:[eE][+-]?(?:\d(?:_?\d)*))?')
|
||||
_datetime_re = re.compile(r'(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(\.\d+)?(?:Z|([+-]\d{2}):(\d{2}))')
|
||||
|
||||
_basicstr_ml_re = re.compile(r'(?:(?:|"|"")[^"\\\000-\011\013-\037])*')
|
||||
_litstr_re = re.compile(r"[^'\000-\037]*")
|
||||
_litstr_ml_re = re.compile(r"(?:(?:|'|'')(?:[^'\000-\011\013-\037]))*")
|
||||
def _p_value(s, object_pairs_hook):
|
||||
pos = s.pos()
|
||||
|
||||
if s.consume('true'):
|
||||
return 'bool', s.last(), True, pos
|
||||
if s.consume('false'):
|
||||
return 'bool', s.last(), False, pos
|
||||
|
||||
if s.consume('"'):
|
||||
if s.consume('""'):
|
||||
r = _p_basicstr_content(s, _basicstr_ml_re)
|
||||
s.expect('"""')
|
||||
else:
|
||||
r = _p_basicstr_content(s, _basicstr_re)
|
||||
s.expect('"')
|
||||
return 'str', r, r, pos
|
||||
|
||||
if s.consume('\''):
|
||||
if s.consume('\'\''):
|
||||
r = s.expect_re(_litstr_ml_re).group(0)
|
||||
s.expect('\'\'\'')
|
||||
else:
|
||||
r = s.expect_re(_litstr_re).group(0)
|
||||
s.expect('\'')
|
||||
return 'str', r, r, pos
|
||||
|
||||
if s.consume_re(_datetime_re):
|
||||
m = s.last()
|
||||
s0 = m.group(0)
|
||||
r = map(int, m.groups()[:6])
|
||||
if m.group(7):
|
||||
micro = float(m.group(7))
|
||||
else:
|
||||
micro = 0
|
||||
|
||||
if m.group(8):
|
||||
g = int(m.group(8), 10) * 60 + int(m.group(9), 10)
|
||||
tz = _TimeZone(datetime.timedelta(0, g * 60))
|
||||
else:
|
||||
tz = _TimeZone(datetime.timedelta(0, 0))
|
||||
|
||||
y, m, d, H, M, S = r
|
||||
dt = datetime.datetime(y, m, d, H, M, S, int(micro * 1000000), tz)
|
||||
return 'datetime', s0, dt, pos
|
||||
|
||||
if s.consume_re(_float_re):
|
||||
m = s.last().group(0)
|
||||
r = m.replace('_','')
|
||||
if '.' in m or 'e' in m or 'E' in m:
|
||||
return 'float', m, float(r), pos
|
||||
else:
|
||||
return 'int', m, int(r, 10), pos
|
||||
|
||||
if s.consume('['):
|
||||
items = []
|
||||
with s:
|
||||
while True:
|
||||
_p_ews(s)
|
||||
items.append(_p_value(s, object_pairs_hook=object_pairs_hook))
|
||||
s.commit()
|
||||
_p_ews(s)
|
||||
s.expect(',')
|
||||
s.commit()
|
||||
_p_ews(s)
|
||||
s.expect(']')
|
||||
return 'array', None, items, pos
|
||||
|
||||
if s.consume('{'):
|
||||
_p_ws(s)
|
||||
items = object_pairs_hook()
|
||||
if not s.consume('}'):
|
||||
k = _p_key(s)
|
||||
_p_ws(s)
|
||||
s.expect('=')
|
||||
_p_ws(s)
|
||||
items[k] = _p_value(s, object_pairs_hook=object_pairs_hook)
|
||||
_p_ws(s)
|
||||
while s.consume(','):
|
||||
_p_ws(s)
|
||||
k = _p_key(s)
|
||||
_p_ws(s)
|
||||
s.expect('=')
|
||||
_p_ws(s)
|
||||
items[k] = _p_value(s, object_pairs_hook=object_pairs_hook)
|
||||
_p_ws(s)
|
||||
s.expect('}')
|
||||
return 'table', None, items, pos
|
||||
|
||||
s.fail()
|
||||
|
||||
def _p_stmt(s, object_pairs_hook):
|
||||
pos = s.pos()
|
||||
if s.consume( '['):
|
||||
is_array = s.consume('[')
|
||||
_p_ws(s)
|
||||
keys = [_p_key(s)]
|
||||
_p_ws(s)
|
||||
while s.consume('.'):
|
||||
_p_ws(s)
|
||||
keys.append(_p_key(s))
|
||||
_p_ws(s)
|
||||
s.expect(']')
|
||||
if is_array:
|
||||
s.expect(']')
|
||||
return 'table_array' if is_array else 'table', keys, pos
|
||||
|
||||
key = _p_key(s)
|
||||
_p_ws(s)
|
||||
s.expect('=')
|
||||
_p_ws(s)
|
||||
value = _p_value(s, object_pairs_hook=object_pairs_hook)
|
||||
return 'kv', (key, value), pos
|
||||
|
||||
_stmtsep_re = re.compile(r'(?:[ \t]*(?:#[^\n]*)?\n)+[ \t]*')
|
||||
def _p_toml(s, object_pairs_hook):
|
||||
stmts = []
|
||||
_p_ews(s)
|
||||
with s:
|
||||
stmts.append(_p_stmt(s, object_pairs_hook=object_pairs_hook))
|
||||
while True:
|
||||
s.commit()
|
||||
s.expect_re(_stmtsep_re)
|
||||
stmts.append(_p_stmt(s, object_pairs_hook=object_pairs_hook))
|
||||
_p_ews(s)
|
||||
s.expect_eof()
|
||||
return stmts
|
||||
|
||||
class _TimeZone(datetime.tzinfo):
|
||||
def __init__(self, offset):
|
||||
self._offset = offset
|
||||
|
||||
def utcoffset(self, dt):
|
||||
return self._offset
|
||||
|
||||
def dst(self, dt):
|
||||
return None
|
||||
|
||||
def tzname(self, dt):
|
||||
m = self._offset.total_seconds() // 60
|
||||
if m < 0:
|
||||
res = '-'
|
||||
m = -m
|
||||
else:
|
||||
res = '+'
|
||||
h = m // 60
|
||||
m = m - h * 60
|
||||
return '{}{:.02}{:.02}'.format(res, h, m)
|
||||
127
mosec/pytoml/writer.py
Normal file
127
mosec/pytoml/writer.py
Normal file
@@ -0,0 +1,127 @@
|
||||
from __future__ import unicode_literals
|
||||
import io, datetime, math, sys
|
||||
|
||||
if sys.version_info[0] == 3:
|
||||
long = int
|
||||
unicode = str
|
||||
|
||||
|
||||
def dumps(obj, sort_keys=False):
|
||||
fout = io.StringIO()
|
||||
dump(obj, fout, sort_keys=sort_keys)
|
||||
return fout.getvalue()
|
||||
|
||||
|
||||
_escapes = {'\n': 'n', '\r': 'r', '\\': '\\', '\t': 't', '\b': 'b', '\f': 'f', '"': '"'}
|
||||
|
||||
|
||||
def _escape_string(s):
|
||||
res = []
|
||||
start = 0
|
||||
|
||||
def flush():
|
||||
if start != i:
|
||||
res.append(s[start:i])
|
||||
return i + 1
|
||||
|
||||
i = 0
|
||||
while i < len(s):
|
||||
c = s[i]
|
||||
if c in '"\\\n\r\t\b\f':
|
||||
start = flush()
|
||||
res.append('\\' + _escapes[c])
|
||||
elif ord(c) < 0x20:
|
||||
start = flush()
|
||||
res.append('\\u%04x' % ord(c))
|
||||
i += 1
|
||||
|
||||
flush()
|
||||
return '"' + ''.join(res) + '"'
|
||||
|
||||
|
||||
def _escape_id(s):
|
||||
if any(not c.isalnum() and c not in '-_' for c in s):
|
||||
return _escape_string(s)
|
||||
return s
|
||||
|
||||
|
||||
def _format_list(v):
|
||||
return '[{0}]'.format(', '.join(_format_value(obj) for obj in v))
|
||||
|
||||
# Formula from:
|
||||
# https://docs.python.org/2/library/datetime.html#datetime.timedelta.total_seconds
|
||||
# Once support for py26 is dropped, this can be replaced by td.total_seconds()
|
||||
def _total_seconds(td):
|
||||
return ((td.microseconds
|
||||
+ (td.seconds + td.days * 24 * 3600) * 10**6) / 10.0**6)
|
||||
|
||||
def _format_value(v):
|
||||
if isinstance(v, bool):
|
||||
return 'true' if v else 'false'
|
||||
if isinstance(v, int) or isinstance(v, long):
|
||||
return unicode(v)
|
||||
if isinstance(v, float):
|
||||
if math.isnan(v) or math.isinf(v):
|
||||
raise ValueError("{0} is not a valid TOML value".format(v))
|
||||
else:
|
||||
return repr(v)
|
||||
elif isinstance(v, unicode) or isinstance(v, bytes):
|
||||
return _escape_string(v)
|
||||
elif isinstance(v, datetime.datetime):
|
||||
offs = v.utcoffset()
|
||||
offs = _total_seconds(offs) // 60 if offs is not None else 0
|
||||
|
||||
if offs == 0:
|
||||
suffix = 'Z'
|
||||
else:
|
||||
if offs > 0:
|
||||
suffix = '+'
|
||||
else:
|
||||
suffix = '-'
|
||||
offs = -offs
|
||||
suffix = '{0}{1:.02}{2:.02}'.format(suffix, offs // 60, offs % 60)
|
||||
|
||||
if v.microsecond:
|
||||
return v.strftime('%Y-%m-%dT%H:%M:%S.%f') + suffix
|
||||
else:
|
||||
return v.strftime('%Y-%m-%dT%H:%M:%S') + suffix
|
||||
elif isinstance(v, list):
|
||||
return _format_list(v)
|
||||
else:
|
||||
raise RuntimeError(v)
|
||||
|
||||
|
||||
def dump(obj, fout, sort_keys=False):
|
||||
tables = [((), obj, False)]
|
||||
|
||||
while tables:
|
||||
name, table, is_array = tables.pop()
|
||||
if name:
|
||||
section_name = '.'.join(_escape_id(c) for c in name)
|
||||
if is_array:
|
||||
fout.write('[[{0}]]\n'.format(section_name))
|
||||
else:
|
||||
fout.write('[{0}]\n'.format(section_name))
|
||||
|
||||
table_keys = sorted(table.keys()) if sort_keys else table.keys()
|
||||
new_tables = []
|
||||
has_kv = False
|
||||
for k in table_keys:
|
||||
v = table[k]
|
||||
if isinstance(v, dict):
|
||||
new_tables.append((name + (k,), v, False))
|
||||
elif isinstance(v, list) and v and all(isinstance(o, dict) for o in v):
|
||||
new_tables.extend((name + (k,), d, True) for d in v)
|
||||
elif v is None:
|
||||
# based on mojombo's comment: https://github.com/toml-lang/toml/issues/146#issuecomment-25019344
|
||||
fout.write(
|
||||
'#{} = null # To use: uncomment and replace null with value\n'.format(_escape_id(k)))
|
||||
has_kv = True
|
||||
else:
|
||||
fout.write('{0} = {1}\n'.format(_escape_id(k), _format_value(v)))
|
||||
has_kv = True
|
||||
|
||||
tables.extend(reversed(new_tables))
|
||||
|
||||
if (name or has_kv) and tables:
|
||||
fout.write('\n')
|
||||
15
mosec/requirement_dist.py
Normal file
15
mosec/requirement_dist.py
Normal file
@@ -0,0 +1,15 @@
|
||||
from pkg_resources import Requirement
|
||||
from mosec import utils
|
||||
|
||||
|
||||
class ReqDist(Requirement):
|
||||
|
||||
def __init__(self, req, dist=None):
|
||||
super(ReqDist, self).__init__(str(req))
|
||||
self.dist = dist
|
||||
|
||||
@property
|
||||
def version(self):
|
||||
if not self.dist:
|
||||
return utils.guess_version(self.key)
|
||||
return self.dist.version
|
||||
102
mosec/requirement_file_parser.py
Normal file
102
mosec/requirement_file_parser.py
Normal file
@@ -0,0 +1,102 @@
|
||||
import re
|
||||
import sys
|
||||
import os
|
||||
from operator import le, lt, gt, ge, eq, ne
|
||||
from mosec import pipfile
|
||||
from mosec import requirements
|
||||
from mosec import setup_file
|
||||
from pkg_resources._vendor.packaging.version import parse as version_parser
|
||||
|
||||
PYTHON_MARKER_REGEX = re.compile(r'python_version\s*(?P<operator>==|<=|=>|>|<)\s*[\'"](?P<python_version>.+?)[\'"]')
|
||||
SYSTEM_MARKER_REGEX = re.compile(r'sys_platform\s*==\s*[\'"](.+)[\'"]')
|
||||
|
||||
|
||||
def satisfies_python_version(parsed_operator, py_version_str):
|
||||
operator_func = {
|
||||
">": gt,
|
||||
"==": eq,
|
||||
"<": lt,
|
||||
"<=": le,
|
||||
">=": ge,
|
||||
'!=': ne,
|
||||
}[parsed_operator]
|
||||
system_py_version = version_parser("{}.{}.{}".format(sys.version_info[0], sys.version_info[1], sys.version_info[2]))
|
||||
required_py_version = version_parser(py_version_str)
|
||||
return operator_func(system_py_version, required_py_version)
|
||||
|
||||
|
||||
def get_markers_text(requirement):
|
||||
if isinstance(requirement, pipfile.PipfileRequirement):
|
||||
return requirement.markers
|
||||
return requirement.line
|
||||
|
||||
|
||||
def matches_python_version(requirement):
|
||||
"""Filter out requirements that should not be installed
|
||||
in this Python version.
|
||||
See: https://www.python.org/dev/peps/pep-0508/#environment-markers
|
||||
"""
|
||||
markers_text = get_markers_text(requirement)
|
||||
if not (markers_text and re.match(".*;.*python_version", markers_text)):
|
||||
return True
|
||||
|
||||
cond_text = markers_text.split(";", 1)[1]
|
||||
|
||||
# Gloss over the 'and' case and return true on the first matching python version
|
||||
|
||||
for sub_exp in re.split("\s*(?:and|or)\s*", cond_text):
|
||||
match = PYTHON_MARKER_REGEX.search(sub_exp)
|
||||
|
||||
if match:
|
||||
match_dict = match.groupdict()
|
||||
|
||||
if len(match_dict) == 2 and satisfies_python_version(
|
||||
match_dict['operator'],
|
||||
match_dict['python_version']
|
||||
):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def matches_environment(requirement):
|
||||
"""Filter out requirements that should not be installed
|
||||
in this environment. Only sys_platform is inspected right now.
|
||||
This should be expanded to include other environment markers.
|
||||
See: https://www.python.org/dev/peps/pep-0508/#environment-markers
|
||||
"""
|
||||
sys_platform = sys.platform.lower()
|
||||
markers_text = get_markers_text(requirement)
|
||||
if markers_text and 'sys_platform' in markers_text:
|
||||
match = SYSTEM_MARKER_REGEX.findall(markers_text)
|
||||
if len(match) > 0:
|
||||
return match[0].lower() == sys_platform
|
||||
return True
|
||||
|
||||
|
||||
def is_testable(requirement):
|
||||
return not requirement.editable and requirement.vcs is None
|
||||
|
||||
|
||||
def get_requirements_list(requirements_file_path):
|
||||
if os.path.basename(requirements_file_path) == 'Pipfile':
|
||||
with open(requirements_file_path, 'r', encoding='utf-8') as f:
|
||||
requirements_data = f.read()
|
||||
parsed_reqs = pipfile.parse(requirements_data)
|
||||
req_list = list(parsed_reqs.get('packages', []))
|
||||
elif os.path.basename(requirements_file_path) == 'setup.py':
|
||||
with open(requirements_file_path, 'r') as f:
|
||||
setup_py_file_content = f.read()
|
||||
requirements_data = setup_file.parse_requirements(setup_py_file_content)
|
||||
req_list = list(requirements.parse(requirements_data))
|
||||
else:
|
||||
# assume this is a requirements.txt formatted file
|
||||
# Note: requirements.txt files are unicode and can be in any encoding.
|
||||
with open(requirements_file_path, 'r') as f:
|
||||
req_list = list(requirements.parse(f))
|
||||
|
||||
req_list = filter(matches_environment, req_list)
|
||||
req_list = filter(is_testable, req_list)
|
||||
req_list = filter(matches_python_version, req_list)
|
||||
req_list = [r for r in req_list if r.name]
|
||||
return req_list
|
||||
29
mosec/requirements/LICENSE.rst
Normal file
29
mosec/requirements/LICENSE.rst
Normal file
@@ -0,0 +1,29 @@
|
||||
License
|
||||
=======
|
||||
|
||||
Requirements Parser is licensed under the BSD license.
|
||||
|
||||
Copyright (c) 2012 - 2013, David Fischer
|
||||
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are met:
|
||||
|
||||
- Redistributions of source code must retain the above copyright notice, this
|
||||
list of conditions and the following disclaimer.
|
||||
- Redistributions in binary form must reproduce the above copyright notice,
|
||||
this list of conditions and the following disclaimer in the documentation
|
||||
and/or other materials provided with the distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY David Fischer ''AS IS'' AND ANY
|
||||
EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
DISCLAIMED. IN NO EVENT SHALL David Fischer BE LIABLE FOR ANY
|
||||
DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
||||
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
57
mosec/requirements/README.rst
Normal file
57
mosec/requirements/README.rst
Normal file
@@ -0,0 +1,57 @@
|
||||
Requirements Parser
|
||||
===================
|
||||
|
||||
|
||||
.. image:: https://travis-ci.org/davidfischer/requirements-parser.svg?branch=master
|
||||
:target: https://travis-ci.org/davidfischer/requirements-parser
|
||||
.. image:: https://coveralls.io/repos/github/davidfischer/requirements-parser/badge.svg?branch=master
|
||||
:target: https://coveralls.io/github/davidfischer/requirements-parser?branch=master
|
||||
.. image:: http://readthedocs.org/projects/requirements-parser/badge/?version=latest
|
||||
:target: http://requirements-parser.readthedocs.io/en/latest/?badge=latest
|
||||
:alt: Documentation Status
|
||||
|
||||
This is a small Python module for parsing Pip_ requirement files.
|
||||
|
||||
The goal is to parse everything in the `Pip requirement file format`_ spec.
|
||||
|
||||
.. _Pip: http://www.pip-installer.org/
|
||||
.. _Pip requirement file format: https://pip.pypa.io/en/stable/reference/pip_install/#requirements-file-format
|
||||
|
||||
|
||||
Installation
|
||||
============
|
||||
|
||||
::
|
||||
|
||||
pip install requirements-parser
|
||||
|
||||
|
||||
Examples
|
||||
========
|
||||
|
||||
Requirements parser can parse a file-like object or a text string.
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
>>> import requirements
|
||||
>>> with open('requirements.txt', 'r') as fd:
|
||||
... for req in requirements.parse(fd):
|
||||
... print(req.name, req.specs)
|
||||
Django [('>=', '1.11'), ('<', '1.12')]
|
||||
six [('==', '1.10.0')]
|
||||
|
||||
It can handle most if not all of the options in requirement files that do
|
||||
not involve traversing the local filesystem. These include:
|
||||
|
||||
* editables (`-e git+https://github.com/toastdriven/pyelasticsearch.git`)
|
||||
* version control URIs
|
||||
* egg hashes and subdirectories (`#egg=django-haystack&subdirectory=setup`)
|
||||
* extras (`DocParser[PDF]`)
|
||||
* URLs
|
||||
|
||||
|
||||
Documentation
|
||||
=============
|
||||
|
||||
For more details and examples, the documentation is available at:
|
||||
http://requirements-parser.readthedocs.io.
|
||||
22
mosec/requirements/__init__.py
Normal file
22
mosec/requirements/__init__.py
Normal file
@@ -0,0 +1,22 @@
|
||||
from .parser import parse # noqa
|
||||
|
||||
_MAJOR = 0
|
||||
_MINOR = 2
|
||||
_PATCH = 0
|
||||
|
||||
|
||||
def version_tuple():
|
||||
'''
|
||||
Returns a 3-tuple of ints that represent the version
|
||||
'''
|
||||
return (_MAJOR, _MINOR, _PATCH)
|
||||
|
||||
|
||||
def version():
|
||||
'''
|
||||
Returns a string representation of the version
|
||||
'''
|
||||
return '%d.%d.%d' % (version_tuple())
|
||||
|
||||
|
||||
__version__ = version()
|
||||
44
mosec/requirements/fragment.py
Normal file
44
mosec/requirements/fragment.py
Normal file
@@ -0,0 +1,44 @@
|
||||
import re
|
||||
|
||||
# Copied from pip
|
||||
# https://github.com/pypa/pip/blob/281eb61b09d87765d7c2b92f6982b3fe76ccb0af/pip/index.py#L947
|
||||
HASH_ALGORITHMS = set(['sha1', 'sha224', 'sha384', 'sha256', 'sha512', 'md5'])
|
||||
|
||||
extras_require_search = re.compile(
|
||||
r'(?P<name>.+)\[(?P<extras>[^\]]+)\]').search
|
||||
|
||||
|
||||
def parse_fragment(fragment_string):
|
||||
"""Takes a fragment string nd returns a dict of the components"""
|
||||
fragment_string = fragment_string.lstrip('#')
|
||||
|
||||
try:
|
||||
return dict(
|
||||
key_value_string.split('=')
|
||||
for key_value_string in fragment_string.split('&')
|
||||
)
|
||||
except ValueError:
|
||||
raise ValueError(
|
||||
'Invalid fragment string {fragment_string}'.format(
|
||||
fragment_string=fragment_string
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def get_hash_info(d):
|
||||
"""Returns the first matching hashlib name and value from a dict"""
|
||||
for key in d.keys():
|
||||
if key.lower() in HASH_ALGORITHMS:
|
||||
return key, d[key]
|
||||
|
||||
return None, None
|
||||
|
||||
|
||||
def parse_extras_require(egg):
|
||||
if egg is not None:
|
||||
match = extras_require_search(egg)
|
||||
if match is not None:
|
||||
name = match.group('name')
|
||||
extras = match.group('extras')
|
||||
return name, [extra.strip() for extra in extras.split(',')]
|
||||
return egg, []
|
||||
50
mosec/requirements/parser.py
Normal file
50
mosec/requirements/parser.py
Normal file
@@ -0,0 +1,50 @@
|
||||
import os
|
||||
import warnings
|
||||
|
||||
from .requirement import Requirement
|
||||
|
||||
|
||||
def parse(reqstr):
|
||||
"""
|
||||
Parse a requirements file into a list of Requirements
|
||||
|
||||
See: pip/req.py:parse_requirements()
|
||||
|
||||
:param reqstr: a string or file like object containing requirements
|
||||
:returns: a *generator* of Requirement objects
|
||||
"""
|
||||
filename = getattr(reqstr, 'name', None)
|
||||
try:
|
||||
# Python 2.x compatibility
|
||||
if not isinstance(reqstr, basestring):
|
||||
reqstr = reqstr.read()
|
||||
except NameError:
|
||||
# Python 3.x only
|
||||
if not isinstance(reqstr, str):
|
||||
reqstr = reqstr.read()
|
||||
|
||||
for line in reqstr.splitlines():
|
||||
line = line.strip()
|
||||
if line == '':
|
||||
continue
|
||||
elif not line or line.startswith('#'):
|
||||
# comments are lines that start with # only
|
||||
continue
|
||||
elif line.startswith('-r') or line.startswith('--requirement'):
|
||||
_, new_filename = line.split()
|
||||
new_file_path = os.path.join(os.path.dirname(filename or '.'),
|
||||
new_filename)
|
||||
with open(new_file_path) as f:
|
||||
for requirement in parse(f):
|
||||
yield requirement
|
||||
elif line.startswith('-f') or line.startswith('--find-links') or \
|
||||
line.startswith('-i') or line.startswith('--index-url') or \
|
||||
line.startswith('--extra-index-url') or \
|
||||
line.startswith('--no-index'):
|
||||
warnings.warn('Private repos not supported. Skipping.')
|
||||
continue
|
||||
elif line.startswith('-Z') or line.startswith('--always-unzip'):
|
||||
warnings.warn('Unused option --always-unzip. Skipping.')
|
||||
continue
|
||||
else:
|
||||
yield Requirement.parse(line, filename)
|
||||
258
mosec/requirements/requirement.py
Normal file
258
mosec/requirements/requirement.py
Normal file
@@ -0,0 +1,258 @@
|
||||
from __future__ import unicode_literals
|
||||
import re
|
||||
import os
|
||||
import warnings
|
||||
from pkg_resources import Requirement as Req
|
||||
|
||||
from .fragment import get_hash_info, parse_fragment, parse_extras_require
|
||||
from .vcs import VCS, VCS_SCHEMES
|
||||
|
||||
|
||||
URI_REGEX = re.compile(
|
||||
r'^(?P<scheme>https?|file|ftps?)://(?P<path>[^#]+)'
|
||||
r'(#(?P<fragment>\S+))?'
|
||||
)
|
||||
|
||||
VCS_REGEX = re.compile(
|
||||
r'^(?P<scheme>{0})://'.format(r'|'.join(
|
||||
[scheme.replace('+', r'\+') for scheme in VCS_SCHEMES])) +
|
||||
r'((?P<login>[^/@]+)@)?'
|
||||
r'(?P<path>[^#@]+)'
|
||||
r'(@(?P<revision>[^#]+))?'
|
||||
r'(#(?P<fragment>\S+))?'
|
||||
)
|
||||
|
||||
# This matches just about everyting
|
||||
LOCAL_REGEX = re.compile(
|
||||
r'^((?P<scheme>file)://)?'
|
||||
r'(?P<path>[^#]+)' +
|
||||
r'(#(?P<fragment>\S+))?'
|
||||
)
|
||||
|
||||
NAME_EQ_REGEX = re.compile(r'name=(?:\'|")(\S+)(?:\'|")')
|
||||
|
||||
|
||||
class Requirement(object):
|
||||
"""
|
||||
Represents a single requirement
|
||||
|
||||
Typically instances of this class are created with ``Requirement.parse``.
|
||||
For local file requirements, there's no verification that the file
|
||||
exists. This class attempts to be *dict-like*.
|
||||
|
||||
See: http://www.pip-installer.org/en/latest/logic.html
|
||||
|
||||
**Members**:
|
||||
|
||||
* ``line`` - the actual requirement line being parsed
|
||||
* ``editable`` - a boolean whether this requirement is "editable"
|
||||
* ``local_file`` - a boolean whether this requirement is a local file/path
|
||||
* ``specifier`` - a boolean whether this requirement used a requirement
|
||||
specifier (eg. "django>=1.5" or "requirements")
|
||||
* ``vcs`` - a string specifying the version control system
|
||||
* ``revision`` - a version control system specifier
|
||||
* ``name`` - the name of the requirement
|
||||
* ``uri`` - the URI if this requirement was specified by URI
|
||||
* ``subdirectory`` - the subdirectory fragment of the URI
|
||||
* ``path`` - the local path to the requirement
|
||||
* ``hash_name`` - the type of hashing algorithm indicated in the line
|
||||
* ``hash`` - the hash value indicated by the requirement line
|
||||
* ``extras`` - a list of extras for this requirement
|
||||
(eg. "mymodule[extra1, extra2]")
|
||||
* ``specs`` - a list of specs for this requirement
|
||||
(eg. "mymodule>1.5,<1.6" => [('>', '1.5'), ('<', '1.6')])
|
||||
* ``provenance`` - a tuple of (file_name, start_line, end_line), line numbers are 1-based
|
||||
"""
|
||||
|
||||
def __init__(self, line):
|
||||
# Do not call this private method
|
||||
self.line = line
|
||||
self.editable = False
|
||||
self.local_file = False
|
||||
self.specifier = False
|
||||
self.vcs = None
|
||||
self.name = None
|
||||
self.subdirectory = None
|
||||
self.uri = None
|
||||
self.path = None
|
||||
self.revision = None
|
||||
self.hash_name = None
|
||||
self.hash = None
|
||||
self.extras = []
|
||||
self.specs = []
|
||||
self.provenance = None
|
||||
|
||||
def __repr__(self):
|
||||
return '<Requirement: "{0}">'.format(self.line)
|
||||
|
||||
def __getitem__(self, key):
|
||||
return getattr(self, key)
|
||||
|
||||
def __eq__(self, other):
|
||||
return all([
|
||||
self.name == other.name,
|
||||
set(self.specs) == set(other.specs),
|
||||
self.editable == other.editable,
|
||||
self.specifier == other.specifier,
|
||||
self.revision == other.revision,
|
||||
self.hash_name == other.hash_name,
|
||||
self.hash == other.hash,
|
||||
set(self.extras) == set(other.extras),
|
||||
])
|
||||
|
||||
def __ne__(self, other):
|
||||
return not self == other
|
||||
|
||||
def keys(self):
|
||||
return self.__dict__.keys()
|
||||
|
||||
@classmethod
|
||||
def parse_editable(cls, line):
|
||||
"""
|
||||
Parses a Requirement from an "editable" requirement which is either
|
||||
a local project path or a VCS project URI.
|
||||
|
||||
See: pip/req.py:from_editable()
|
||||
|
||||
:param line: an "editable" requirement
|
||||
:returns: a Requirement instance for the given line
|
||||
:raises: ValueError on an invalid requirement
|
||||
"""
|
||||
|
||||
req = cls('-e {0}'.format(line))
|
||||
req.editable = True
|
||||
|
||||
if ' #' in line:
|
||||
line = line[:line.find(' #')]
|
||||
|
||||
vcs_match = VCS_REGEX.match(line)
|
||||
local_match = LOCAL_REGEX.match(line)
|
||||
|
||||
if vcs_match is not None:
|
||||
groups = vcs_match.groupdict()
|
||||
if groups.get('login'):
|
||||
req.uri = '{scheme}://{login}@{path}'.format(**groups)
|
||||
else:
|
||||
req.uri = '{scheme}://{path}'.format(**groups)
|
||||
req.revision = groups['revision']
|
||||
if groups['fragment']:
|
||||
fragment = parse_fragment(groups['fragment'])
|
||||
egg = fragment.get('egg')
|
||||
req.name, req.extras = parse_extras_require(egg)
|
||||
req.hash_name, req.hash = get_hash_info(fragment)
|
||||
req.subdirectory = fragment.get('subdirectory')
|
||||
for vcs in VCS:
|
||||
if req.uri.startswith(vcs):
|
||||
req.vcs = vcs
|
||||
else:
|
||||
assert local_match is not None, 'This should match everything'
|
||||
groups = local_match.groupdict()
|
||||
req.local_file = True
|
||||
if groups['fragment']:
|
||||
fragment = parse_fragment(groups['fragment'])
|
||||
egg = fragment.get('egg')
|
||||
req.name, req.extras = parse_extras_require(egg)
|
||||
req.hash_name, req.hash = get_hash_info(fragment)
|
||||
req.subdirectory = fragment.get('subdirectory')
|
||||
req.path = groups['path']
|
||||
|
||||
return req
|
||||
|
||||
@classmethod
|
||||
def parse_line(cls, line, filename=''):
|
||||
"""
|
||||
Parses a Requirement from a non-editable requirement.
|
||||
|
||||
See: pip/req.py:from_line()
|
||||
|
||||
:param line: a "non-editable" requirement
|
||||
:param filename: (optional) where/to/requirements.txt
|
||||
:returns: a Requirement instance for the given line
|
||||
:raises: ValueError on an invalid requirement
|
||||
"""
|
||||
|
||||
req = cls(line)
|
||||
|
||||
vcs_match = VCS_REGEX.match(line)
|
||||
uri_match = URI_REGEX.match(line)
|
||||
local_match = LOCAL_REGEX.match(line)
|
||||
|
||||
if vcs_match is not None:
|
||||
groups = vcs_match.groupdict()
|
||||
if groups.get('login'):
|
||||
req.uri = '{scheme}://{login}@{path}'.format(**groups)
|
||||
else:
|
||||
req.uri = '{scheme}://{path}'.format(**groups)
|
||||
req.revision = groups['revision']
|
||||
if groups['fragment']:
|
||||
fragment = parse_fragment(groups['fragment'])
|
||||
egg = fragment.get('egg')
|
||||
req.name, req.extras = parse_extras_require(egg)
|
||||
req.hash_name, req.hash = get_hash_info(fragment)
|
||||
req.subdirectory = fragment.get('subdirectory')
|
||||
for vcs in VCS:
|
||||
if req.uri.startswith(vcs):
|
||||
req.vcs = vcs
|
||||
elif uri_match is not None:
|
||||
groups = uri_match.groupdict()
|
||||
req.uri = '{scheme}://{path}'.format(**groups)
|
||||
if groups['fragment']:
|
||||
fragment = parse_fragment(groups['fragment'])
|
||||
egg = fragment.get('egg')
|
||||
req.name, req.extras = parse_extras_require(egg)
|
||||
req.hash_name, req.hash = get_hash_info(fragment)
|
||||
req.subdirectory = fragment.get('subdirectory')
|
||||
if groups['scheme'] == 'file':
|
||||
req.local_file = True
|
||||
elif '#egg=' in line:
|
||||
# Assume a local file match
|
||||
assert local_match is not None, 'This should match everything'
|
||||
groups = local_match.groupdict()
|
||||
req.local_file = True
|
||||
if groups['fragment']:
|
||||
fragment = parse_fragment(groups['fragment'])
|
||||
egg = fragment.get('egg')
|
||||
name, extras = parse_extras_require(egg)
|
||||
req.name = fragment.get('egg')
|
||||
req.hash_name, req.hash = get_hash_info(fragment)
|
||||
req.subdirectory = fragment.get('subdirectory')
|
||||
req.path = groups['path']
|
||||
elif os.path.isfile(os.path.join(os.path.dirname(filename or '.'), line, "setup.py")):
|
||||
setup_file = open(os.path.join(os.path.dirname(filename or '.'), line, "setup.py"), "r")
|
||||
setup_content = setup_file.read()
|
||||
setup_file.close()
|
||||
name_search = NAME_EQ_REGEX.search(setup_content)
|
||||
if name_search:
|
||||
req.name = name_search.group(1)
|
||||
req.local_file = True
|
||||
else:
|
||||
# This is a requirement specifier.
|
||||
# Delegate to pkg_resources and hope for the best
|
||||
req.specifier = True
|
||||
try:
|
||||
pkg_req = Req.parse(line)
|
||||
req.name = pkg_req.unsafe_name
|
||||
req.extras = list(pkg_req.extras)
|
||||
req.specs = pkg_req.specs
|
||||
except Exception as e:
|
||||
warnings.warn("unkonwn line: {}".format(line))
|
||||
return req
|
||||
|
||||
@classmethod
|
||||
def parse(cls, line, filename=''):
|
||||
"""
|
||||
Parses a Requirement from a line of a requirement file.
|
||||
|
||||
:param line: a line of a requirement file
|
||||
:param filename: (optional) where/to/requirements.txt
|
||||
:returns: a Requirement instance for the given line
|
||||
:raises: ValueError on an invalid requirement
|
||||
"""
|
||||
|
||||
if line.startswith('-e') or line.startswith('--editable'):
|
||||
# Editable installs are either a local project path
|
||||
# or a VCS project URI
|
||||
return cls.parse_editable(
|
||||
re.sub(r'^(-e|--editable=?)\s*', '', line))
|
||||
|
||||
return cls.parse_line(line, filename)
|
||||
30
mosec/requirements/vcs.py
Normal file
30
mosec/requirements/vcs.py
Normal file
@@ -0,0 +1,30 @@
|
||||
from __future__ import unicode_literals
|
||||
|
||||
VCS = [
|
||||
'git',
|
||||
'hg',
|
||||
'svn',
|
||||
'bzr',
|
||||
]
|
||||
|
||||
VCS_SCHEMES = [
|
||||
'git',
|
||||
'git+https',
|
||||
'git+ssh',
|
||||
'git+git',
|
||||
'hg+http',
|
||||
'hg+https',
|
||||
'hg+static-http',
|
||||
'hg+ssh',
|
||||
'svn',
|
||||
'svn+svn',
|
||||
'svn+http',
|
||||
'svn+https',
|
||||
'svn+ssh',
|
||||
'bzr+http',
|
||||
'bzr+https',
|
||||
'bzr+ssh',
|
||||
'bzr+sftp',
|
||||
'bzr+ftp',
|
||||
'bzr+lp',
|
||||
]
|
||||
40
mosec/setup_file.py
Normal file
40
mosec/setup_file.py
Normal file
@@ -0,0 +1,40 @@
|
||||
import distutils.core
|
||||
import re
|
||||
import setuptools
|
||||
|
||||
|
||||
def parse(setup_py_content):
|
||||
"""Parse a setup.py file and extract the arguments passed to the setup method"""
|
||||
|
||||
# Make the setup method return the arguments that are passed to it
|
||||
def _save_passed_args(**kwargs):
|
||||
return kwargs
|
||||
|
||||
distutils.core.setup = _save_passed_args
|
||||
setuptools.setup = _save_passed_args
|
||||
setup_py_content = setup_py_content.replace("setup(", "passed_arguments = setup(")
|
||||
|
||||
# Fetch the arguments that were passed to the setup.py
|
||||
exec(setup_py_content, globals())
|
||||
return globals()["passed_arguments"]
|
||||
|
||||
|
||||
def parse_name_and_version(setup_py_content):
|
||||
"""Extract the name and version from a setup.py file"""
|
||||
passed_arguments = parse(setup_py_content)
|
||||
return passed_arguments.get("name"), passed_arguments.get("version")
|
||||
|
||||
|
||||
def get_provenance(setup_py_content):
|
||||
"""Provenance for a setup.py file is the index of the line that contains `install_requires`"""
|
||||
for index, line in enumerate(setup_py_content.splitlines()):
|
||||
if re.search(r"(packages|install_requires)\s*=", line):
|
||||
return index + 1
|
||||
return -1
|
||||
|
||||
|
||||
def parse_requirements(setup_py_content):
|
||||
"""Extract the dependencies from a setup.py file"""
|
||||
passed_arguments = parse(setup_py_content)
|
||||
requirements = passed_arguments.get("install_requires", passed_arguments.get("packages", []))
|
||||
return "\n".join(requirements)
|
||||
34
mosec/utils.py
Normal file
34
mosec/utils.py
Normal file
@@ -0,0 +1,34 @@
|
||||
import sys
|
||||
import re
|
||||
from importlib import import_module
|
||||
|
||||
|
||||
def canonicalize_dist_name(name):
|
||||
# https://packaging.python.org/guides/distributing-packages-using-setuptools/#name
|
||||
name = name.lower().replace('-', '.').replace('_', '.')
|
||||
name = re.sub(r'\.+', '.', name)
|
||||
return name
|
||||
|
||||
|
||||
def guess_version(pkg_key, default='?'):
|
||||
"""Guess the version of a pkg when pip doesn't provide it
|
||||
:param str pkg_key: key of the package
|
||||
:param str default: default version to return if unable to find
|
||||
:returns: version
|
||||
:rtype: string
|
||||
"""
|
||||
try:
|
||||
m = import_module(pkg_key)
|
||||
except ImportError:
|
||||
return default
|
||||
else:
|
||||
return getattr(m, '__version__', default)
|
||||
|
||||
|
||||
def is_string(obj):
|
||||
"""Check whether an object is a string"""
|
||||
if sys.version_info < (3,):
|
||||
# Python 2.x only
|
||||
return isinstance(obj, basestring)
|
||||
else:
|
||||
return isinstance(obj, str)
|
||||
Reference in New Issue
Block a user