From 09da953f8b0e1597c7e2d996859efc981013a71f Mon Sep 17 00:00:00 2001 From: retanoj <> Date: Tue, 28 Jul 2020 15:42:47 +0800 Subject: [PATCH] Initial commit --- .gitignore | 198 ++++++++++++++++ CHANGES | 40 ++++ LICENSE | 13 + README.md | 39 +++ mosec/__init__.py | 0 mosec/colorama/LICENSE.txt | 27 +++ mosec/colorama/README.rst | 380 ++++++++++++++++++++++++++++++ mosec/colorama/__init__.py | 6 + mosec/colorama/ansi.py | 102 ++++++++ mosec/colorama/ansitowin32.py | 258 ++++++++++++++++++++ mosec/colorama/initialise.py | 80 +++++++ mosec/colorama/win32.py | 152 ++++++++++++ mosec/colorama/winterm.py | 169 +++++++++++++ mosec/mosec_log_helper.py | 30 +++ mosec/pip_resolve.py | 265 +++++++++++++++++++++ mosec/pipfile.py | 63 +++++ mosec/pytoml/LICENSE | 16 ++ mosec/pytoml/README.txt | 9 + mosec/pytoml/__init__.py | 3 + mosec/pytoml/core.py | 13 + mosec/pytoml/parser.py | 374 +++++++++++++++++++++++++++++ mosec/pytoml/writer.py | 127 ++++++++++ mosec/requirement_dist.py | 15 ++ mosec/requirement_file_parser.py | 102 ++++++++ mosec/requirements/LICENSE.rst | 29 +++ mosec/requirements/README.rst | 57 +++++ mosec/requirements/__init__.py | 22 ++ mosec/requirements/fragment.py | 44 ++++ mosec/requirements/parser.py | 50 ++++ mosec/requirements/requirement.py | 258 ++++++++++++++++++++ mosec/requirements/vcs.py | 30 +++ mosec/setup_file.py | 40 ++++ mosec/utils.py | 34 +++ setup.py | 32 +++ 34 files changed, 3077 insertions(+) create mode 100644 .gitignore create mode 100644 CHANGES create mode 100644 LICENSE create mode 100644 README.md create mode 100644 mosec/__init__.py create mode 100644 mosec/colorama/LICENSE.txt create mode 100644 mosec/colorama/README.rst create mode 100644 mosec/colorama/__init__.py create mode 100644 mosec/colorama/ansi.py create mode 100644 mosec/colorama/ansitowin32.py create mode 100644 mosec/colorama/initialise.py create mode 100644 mosec/colorama/win32.py create mode 100644 mosec/colorama/winterm.py create mode 100644 mosec/mosec_log_helper.py create mode 100644 mosec/pip_resolve.py create mode 100644 mosec/pipfile.py create mode 100644 mosec/pytoml/LICENSE create mode 100644 mosec/pytoml/README.txt create mode 100644 mosec/pytoml/__init__.py create mode 100644 mosec/pytoml/core.py create mode 100644 mosec/pytoml/parser.py create mode 100644 mosec/pytoml/writer.py create mode 100644 mosec/requirement_dist.py create mode 100644 mosec/requirement_file_parser.py create mode 100644 mosec/requirements/LICENSE.rst create mode 100644 mosec/requirements/README.rst create mode 100644 mosec/requirements/__init__.py create mode 100644 mosec/requirements/fragment.py create mode 100644 mosec/requirements/parser.py create mode 100644 mosec/requirements/requirement.py create mode 100644 mosec/requirements/vcs.py create mode 100644 mosec/setup_file.py create mode 100644 mosec/utils.py create mode 100644 setup.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..fd7c043 --- /dev/null +++ b/.gitignore @@ -0,0 +1,198 @@ + +# Created by https://www.gitignore.io/api/python,pycharm+all +# Edit at https://www.gitignore.io/?templates=python,pycharm+all + +### PyCharm+all ### +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/usage.statistics.xml +.idea/**/dictionaries +.idea/**/shelf + +# Generated files +.idea/**/contentModel.xml + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# Gradle and Maven with auto-import +# When using Gradle or Maven with auto-import, you should exclude module files, +# since they will be recreated, and may cause churn. Uncomment if using +# auto-import. +# .idea/modules.xml +# .idea/*.iml +# .idea/modules +# *.iml +# *.ipr + +# CMake +cmake-build-*/ + +# Mongo Explorer plugin +.idea/**/mongoSettings.xml + +# File-based project format +*.iws + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# Editor-based Rest Client +.idea/httpRequests + +# Android studio 3.1+ serialized cache file +.idea/caches/build_file_checksums.ser + +### PyCharm+all Patch ### +# Ignores the whole .idea folder and all .iml files +# See https://github.com/joeblau/gitignore.io/issues/186 and https://github.com/joeblau/gitignore.io/issues/360 + +.idea/ + +# Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-249601023 + +*.iml +modules.xml +.idea/misc.xml +*.ipr + +# Sonarlint plugin +.idea/sonarlint + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# Mr Developer +.mr.developer.cfg +.project +.pydevproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# End of https://www.gitignore.io/api/python,pycharm+all + +venv/ diff --git a/CHANGES b/CHANGES new file mode 100644 index 0000000..3341154 --- /dev/null +++ b/CHANGES @@ -0,0 +1,40 @@ +Changelog + +========= + +Version 1.1.0 + +- feature remove Package class and subclass +- feature add LogHelper +- feature add ReqDist + +Version 1.0.6 + +- feature default throw Exception when found vulnerable && parameter support + +Version 1.0.5 + +- feature change color log + +Version 1.0.4 + +- feature color log + +Version 1.0.3 + +- feature replace urllib3 with urllib + +Version 1.0.2 + +- bugfix Requirement object has provenance property +- feature replace requests dependency with urllib3 + +Version 1.0.1 + +- bugfix support requierments.txt with local dir dependency +- upgrade requirements upgrade to 0.2.0 +- feature add exit code + +Version 1.0.0 + +- Init \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..1bcd69a --- /dev/null +++ b/LICENSE @@ -0,0 +1,13 @@ +Copyright 2020 momosecurity. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..29f4b4a --- /dev/null +++ b/README.md @@ -0,0 +1,39 @@ +# MOSEC-PIP-PLUGIN + +用于检测python项目的第三方依赖组件是否存在安全漏洞。 + +该项目是基于 [snyk-python-plugin](https://github.com/snyk/snyk-python-plugin.git) 的二次开发。 + +## 版本支持 + +Python 3.x + +## 安装 + +``` +pip install git+https://github.com/momosecurity/mosec-pip-plugin.git +``` + +## 使用 + +首先运行 [MOSEC-X-PLUGIN Backend](https://github.com/momosecurity/mosec-x-plugin-backend.git) + +``` +> cd your_python_project_dir/ +> mosec requirements.txt --endpoint http://127.0.0.1:9000/api/plugin --only-provenance + +// 或 +> mosec setup.py --endpoint http://127.0.0.1:9000/api/plugin --only-provenance +``` + +## 卸载 + +``` +> pip uninstall mosec-pip-plugin +``` + +## 开发 + +#### Pycharm 调试 mosec-pip-plugin + +程序入口位于`mosec/pip_resolve.py`文件的`main()`函数 \ No newline at end of file diff --git a/mosec/__init__.py b/mosec/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/mosec/colorama/LICENSE.txt b/mosec/colorama/LICENSE.txt new file mode 100644 index 0000000..3105888 --- /dev/null +++ b/mosec/colorama/LICENSE.txt @@ -0,0 +1,27 @@ +Copyright (c) 2010 Jonathan Hartley +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holders, nor those of its contributors + may be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/mosec/colorama/README.rst b/mosec/colorama/README.rst new file mode 100644 index 0000000..9a5f7d0 --- /dev/null +++ b/mosec/colorama/README.rst @@ -0,0 +1,380 @@ +.. image:: https://img.shields.io/pypi/v/colorama.svg + :target: https://pypi.org/project/colorama/ + :alt: Latest Version + +.. image:: https://img.shields.io/pypi/pyversions/colorama.svg + :target: https://pypi.org/project/colorama/ + :alt: Supported Python versions + +.. image:: https://travis-ci.org/tartley/colorama.svg?branch=master + :target: https://travis-ci.org/tartley/colorama + :alt: Build Status + +Download and docs: + https://pypi.org/project/colorama/ +Source code & Development: + https://github.com/tartley/colorama +Colorama for Enterprise: + https://github.com/tartley/colorama/blob/master/ENTERPRISE.md + +Description +=========== + +Makes ANSI escape character sequences (for producing colored terminal text and +cursor positioning) work under MS Windows. + +ANSI escape character sequences have long been used to produce colored terminal +text and cursor positioning on Unix and Macs. Colorama makes this work on +Windows, too, by wrapping ``stdout``, stripping ANSI sequences it finds (which +would appear as gobbledygook in the output), and converting them into the +appropriate win32 calls to modify the state of the terminal. On other platforms, +Colorama does nothing. + +Colorama also provides some shortcuts to help generate ANSI sequences +but works fine in conjunction with any other ANSI sequence generation library, +such as the venerable Termcolor (https://pypi.org/project/termcolor/) +or the fabulous Blessings (https://pypi.org/project/blessings/). + +This has the upshot of providing a simple cross-platform API for printing +colored terminal text from Python, and has the happy side-effect that existing +applications or libraries which use ANSI sequences to produce colored output on +Linux or Macs can now also work on Windows, simply by calling +``colorama.init()``. + +An alternative approach is to install ``ansi.sys`` on Windows machines, which +provides the same behaviour for all applications running in terminals. Colorama +is intended for situations where that isn't easy (e.g., maybe your app doesn't +have an installer.) + +Demo scripts in the source code repository print some colored text using +ANSI sequences. Compare their output under Gnome-terminal's built in ANSI +handling, versus on Windows Command-Prompt using Colorama: + +.. image:: https://github.com/tartley/colorama/raw/master/screenshots/ubuntu-demo.png + :width: 661 + :height: 357 + :alt: ANSI sequences on Ubuntu under gnome-terminal. + +.. image:: https://github.com/tartley/colorama/raw/master/screenshots/windows-demo.png + :width: 668 + :height: 325 + :alt: Same ANSI sequences on Windows, using Colorama. + +These screengrabs show that, on Windows, Colorama does not support ANSI 'dim +text'; it looks the same as 'normal text'. + + +License +======= + +Copyright Jonathan Hartley & Arnon Yaari, 2013. BSD 3-Clause license; see LICENSE file. + + +Dependencies +============ + +None, other than Python. Tested on Python 2.7, 3.5, 3.6, 3.7 and 3.8. + +Usage +===== + +Initialisation +-------------- + +Applications should initialise Colorama using: + +.. code-block:: python + + from colorama import init + init() + +On Windows, calling ``init()`` will filter ANSI escape sequences out of any +text sent to ``stdout`` or ``stderr``, and replace them with equivalent Win32 +calls. + +On other platforms, calling ``init()`` has no effect (unless you request other +optional functionality; see "Init Keyword Args", below). By design, this permits +applications to call ``init()`` unconditionally on all platforms, after which +ANSI output should just work. + +To stop using colorama before your program exits, simply call ``deinit()``. +This will restore ``stdout`` and ``stderr`` to their original values, so that +Colorama is disabled. To resume using Colorama again, call ``reinit()``; it is +cheaper than calling ``init()`` again (but does the same thing). + + +Colored Output +-------------- + +Cross-platform printing of colored text can then be done using Colorama's +constant shorthand for ANSI escape sequences: + +.. code-block:: python + + from colorama import Fore, Back, Style + print(Fore.RED + 'some red text') + print(Back.GREEN + 'and with a green background') + print(Style.DIM + 'and in dim text') + print(Style.RESET_ALL) + print('back to normal now') + +...or simply by manually printing ANSI sequences from your own code: + +.. code-block:: python + + print('\033[31m' + 'some red text') + print('\033[39m') # and reset to default color + +...or, Colorama can be used happily in conjunction with existing ANSI libraries +such as Termcolor: + +.. code-block:: python + + from colorama import init + from termcolor import colored + + # use Colorama to make Termcolor work on Windows too + init() + + # then use Termcolor for all colored text output + print(colored('Hello, World!', 'green', 'on_red')) + +Available formatting constants are:: + + Fore: BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE, RESET. + Back: BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE, RESET. + Style: DIM, NORMAL, BRIGHT, RESET_ALL + +``Style.RESET_ALL`` resets foreground, background, and brightness. Colorama will +perform this reset automatically on program exit. + + +Cursor Positioning +------------------ + +ANSI codes to reposition the cursor are supported. See ``demos/demo06.py`` for +an example of how to generate them. + + +Init Keyword Args +----------------- + +``init()`` accepts some ``**kwargs`` to override default behaviour. + +init(autoreset=False): + If you find yourself repeatedly sending reset sequences to turn off color + changes at the end of every print, then ``init(autoreset=True)`` will + automate that: + + .. code-block:: python + + from colorama import init + init(autoreset=True) + print(Fore.RED + 'some red text') + print('automatically back to default color again') + +init(strip=None): + Pass ``True`` or ``False`` to override whether ansi codes should be + stripped from the output. The default behaviour is to strip if on Windows + or if output is redirected (not a tty). + +init(convert=None): + Pass ``True`` or ``False`` to override whether to convert ANSI codes in the + output into win32 calls. The default behaviour is to convert if on Windows + and output is to a tty (terminal). + +init(wrap=True): + On Windows, colorama works by replacing ``sys.stdout`` and ``sys.stderr`` + with proxy objects, which override the ``.write()`` method to do their work. + If this wrapping causes you problems, then this can be disabled by passing + ``init(wrap=False)``. The default behaviour is to wrap if ``autoreset`` or + ``strip`` or ``convert`` are True. + + When wrapping is disabled, colored printing on non-Windows platforms will + continue to work as normal. To do cross-platform colored output, you can + use Colorama's ``AnsiToWin32`` proxy directly: + + .. code-block:: python + + import sys + from colorama import init, AnsiToWin32 + init(wrap=False) + stream = AnsiToWin32(sys.stderr).stream + + # Python 2 + print >>stream, Fore.BLUE + 'blue text on stderr' + + # Python 3 + print(Fore.BLUE + 'blue text on stderr', file=stream) + + +Installation +======================= +colorama is currently installable from PyPI: + + pip install colorama + +colorama also can be installed by the conda package manager: + + conda install -c anaconda colorama + + +Status & Known Problems +======================= + +I've personally only tested it on Windows XP (CMD, Console2), Ubuntu +(gnome-terminal, xterm), and OS X. + +Some presumably valid ANSI sequences aren't recognised (see details below), +but to my knowledge nobody has yet complained about this. Puzzling. + +See outstanding issues and wishlist: +https://github.com/tartley/colorama/issues + +If anything doesn't work for you, or doesn't do what you expected or hoped for, +I'd love to hear about it on that issues list, would be delighted by patches, +and would be happy to grant commit access to anyone who submits a working patch +or two. + + +Recognised ANSI Sequences +========================= + +ANSI sequences generally take the form: + + ESC [ ; ... + +Where ```` is an integer, and ```` is a single letter. Zero or +more params are passed to a ````. If no params are passed, it is +generally synonymous with passing a single zero. No spaces exist in the +sequence; they have been inserted here simply to read more easily. + +The only ANSI sequences that colorama converts into win32 calls are:: + + ESC [ 0 m # reset all (colors and brightness) + ESC [ 1 m # bright + ESC [ 2 m # dim (looks same as normal brightness) + ESC [ 22 m # normal brightness + + # FOREGROUND: + ESC [ 30 m # black + ESC [ 31 m # red + ESC [ 32 m # green + ESC [ 33 m # yellow + ESC [ 34 m # blue + ESC [ 35 m # magenta + ESC [ 36 m # cyan + ESC [ 37 m # white + ESC [ 39 m # reset + + # BACKGROUND + ESC [ 40 m # black + ESC [ 41 m # red + ESC [ 42 m # green + ESC [ 43 m # yellow + ESC [ 44 m # blue + ESC [ 45 m # magenta + ESC [ 46 m # cyan + ESC [ 47 m # white + ESC [ 49 m # reset + + # cursor positioning + ESC [ y;x H # position cursor at x across, y down + ESC [ y;x f # position cursor at x across, y down + ESC [ n A # move cursor n lines up + ESC [ n B # move cursor n lines down + ESC [ n C # move cursor n characters forward + ESC [ n D # move cursor n characters backward + + # clear the screen + ESC [ mode J # clear the screen + + # clear the line + ESC [ mode K # clear the line + +Multiple numeric params to the ``'m'`` command can be combined into a single +sequence:: + + ESC [ 36 ; 45 ; 1 m # bright cyan text on magenta background + +All other ANSI sequences of the form ``ESC [ ; ... `` +are silently stripped from the output on Windows. + +Any other form of ANSI sequence, such as single-character codes or alternative +initial characters, are not recognised or stripped. It would be cool to add +them though. Let me know if it would be useful for you, via the Issues on +GitHub. + + +Development +=========== + +Help and fixes welcome! + +Running tests requires: + +- Michael Foord's ``mock`` module to be installed on Python < 3.3. +- Tests are written using 2010-era updates to ``unittest`` + +To run tests:: + + python -m unittest discover -p *_test.py + +This, like a few other handy commands, is captured in a ``Makefile``. + +If you use nose to run the tests, you must pass the ``-s`` flag; otherwise, +``nosetests`` applies its own proxy to ``stdout``, which confuses the unit +tests. + + +Professional support +==================== + +.. |tideliftlogo| image:: https://cdn2.hubspot.net/hubfs/4008838/website/logos/logos_for_download/Tidelift_primary-shorthand-logo.png + :alt: Tidelift + :target: https://tidelift.com/subscription/pkg/pypi-colorama?utm_source=pypi-colorama&utm_medium=referral&utm_campaign=readme + +.. list-table:: + :widths: 10 100 + + * - |tideliftlogo| + - Professional support for colorama is available as part of the + `Tidelift Subscription`_. + Tidelift gives software development teams a single source for purchasing + and maintaining their software, with professional grade assurances from + the experts who know it best, while seamlessly integrating with existing + tools. + +.. _Tidelift Subscription: https://tidelift.com/subscription/pkg/pypi-colorama?utm_source=pypi-colorama&utm_medium=referral&utm_campaign=readme + + +Thanks +====== +* Marc Schlaich (schlamar) for a ``setup.py`` fix for Python2.5. +* Marc Abramowitz, reported & fixed a crash on exit with closed ``stdout``, + providing a solution to issue #7's setuptools/distutils debate, + and other fixes. +* User 'eryksun', for guidance on correctly instantiating ``ctypes.windll``. +* Matthew McCormick for politely pointing out a longstanding crash on non-Win. +* Ben Hoyt, for a magnificent fix under 64-bit Windows. +* Jesse at Empty Square for submitting a fix for examples in the README. +* User 'jamessp', an observant documentation fix for cursor positioning. +* User 'vaal1239', Dave Mckee & Lackner Kristof for a tiny but much-needed Win7 + fix. +* Julien Stuyck, for wisely suggesting Python3 compatible updates to README. +* Daniel Griffith for multiple fabulous patches. +* Oscar Lesta for a valuable fix to stop ANSI chars being sent to non-tty + output. +* Roger Binns, for many suggestions, valuable feedback, & bug reports. +* Tim Golden for thought and much appreciated feedback on the initial idea. +* User 'Zearin' for updates to the README file. +* John Szakmeister for adding support for light colors +* Charles Merriam for adding documentation to demos +* Jurko for a fix on 64-bit Windows CPython2.5 w/o ctypes +* Florian Bruhin for a fix when stdout or stderr are None +* Thomas Weininger for fixing ValueError on Windows +* Remi Rampin for better Github integration and fixes to the README file +* Simeon Visser for closing a file handle using 'with' and updating classifiers + to include Python 3.3 and 3.4 +* Andy Neff for fixing RESET of LIGHT_EX colors. +* Jonathan Hartley for the initial idea and implementation. diff --git a/mosec/colorama/__init__.py b/mosec/colorama/__init__.py new file mode 100644 index 0000000..b149ed7 --- /dev/null +++ b/mosec/colorama/__init__.py @@ -0,0 +1,6 @@ +# Copyright Jonathan Hartley 2013. BSD 3-Clause license, see LICENSE file. +from .initialise import init, deinit, reinit, colorama_text +from .ansi import Fore, Back, Style, Cursor +from .ansitowin32 import AnsiToWin32 + +__version__ = '0.4.4' diff --git a/mosec/colorama/ansi.py b/mosec/colorama/ansi.py new file mode 100644 index 0000000..11ec695 --- /dev/null +++ b/mosec/colorama/ansi.py @@ -0,0 +1,102 @@ +# Copyright Jonathan Hartley 2013. BSD 3-Clause license, see LICENSE file. +''' +This module generates ANSI character codes to printing colors to terminals. +See: http://en.wikipedia.org/wiki/ANSI_escape_code +''' + +CSI = '\033[' +OSC = '\033]' +BEL = '\a' + + +def code_to_chars(code): + return CSI + str(code) + 'm' + +def set_title(title): + return OSC + '2;' + title + BEL + +def clear_screen(mode=2): + return CSI + str(mode) + 'J' + +def clear_line(mode=2): + return CSI + str(mode) + 'K' + + +class AnsiCodes(object): + def __init__(self): + # the subclasses declare class attributes which are numbers. + # Upon instantiation we define instance attributes, which are the same + # as the class attributes but wrapped with the ANSI escape sequence + for name in dir(self): + if not name.startswith('_'): + value = getattr(self, name) + setattr(self, name, code_to_chars(value)) + + +class AnsiCursor(object): + def UP(self, n=1): + return CSI + str(n) + 'A' + def DOWN(self, n=1): + return CSI + str(n) + 'B' + def FORWARD(self, n=1): + return CSI + str(n) + 'C' + def BACK(self, n=1): + return CSI + str(n) + 'D' + def POS(self, x=1, y=1): + return CSI + str(y) + ';' + str(x) + 'H' + + +class AnsiFore(AnsiCodes): + BLACK = 30 + RED = 31 + GREEN = 32 + YELLOW = 33 + BLUE = 34 + MAGENTA = 35 + CYAN = 36 + WHITE = 37 + RESET = 39 + + # These are fairly well supported, but not part of the standard. + LIGHTBLACK_EX = 90 + LIGHTRED_EX = 91 + LIGHTGREEN_EX = 92 + LIGHTYELLOW_EX = 93 + LIGHTBLUE_EX = 94 + LIGHTMAGENTA_EX = 95 + LIGHTCYAN_EX = 96 + LIGHTWHITE_EX = 97 + + +class AnsiBack(AnsiCodes): + BLACK = 40 + RED = 41 + GREEN = 42 + YELLOW = 43 + BLUE = 44 + MAGENTA = 45 + CYAN = 46 + WHITE = 47 + RESET = 49 + + # These are fairly well supported, but not part of the standard. + LIGHTBLACK_EX = 100 + LIGHTRED_EX = 101 + LIGHTGREEN_EX = 102 + LIGHTYELLOW_EX = 103 + LIGHTBLUE_EX = 104 + LIGHTMAGENTA_EX = 105 + LIGHTCYAN_EX = 106 + LIGHTWHITE_EX = 107 + + +class AnsiStyle(AnsiCodes): + BRIGHT = 1 + DIM = 2 + NORMAL = 22 + RESET_ALL = 0 + +Fore = AnsiFore() +Back = AnsiBack() +Style = AnsiStyle() +Cursor = AnsiCursor() diff --git a/mosec/colorama/ansitowin32.py b/mosec/colorama/ansitowin32.py new file mode 100644 index 0000000..6039a05 --- /dev/null +++ b/mosec/colorama/ansitowin32.py @@ -0,0 +1,258 @@ +# Copyright Jonathan Hartley 2013. BSD 3-Clause license, see LICENSE file. +import re +import sys +import os + +from .ansi import AnsiFore, AnsiBack, AnsiStyle, Style, BEL +from .winterm import WinTerm, WinColor, WinStyle +from .win32 import windll, winapi_test + + +winterm = None +if windll is not None: + winterm = WinTerm() + + +class StreamWrapper(object): + ''' + Wraps a stream (such as stdout), acting as a transparent proxy for all + attribute access apart from method 'write()', which is delegated to our + Converter instance. + ''' + def __init__(self, wrapped, converter): + # double-underscore everything to prevent clashes with names of + # attributes on the wrapped stream object. + self.__wrapped = wrapped + self.__convertor = converter + + def __getattr__(self, name): + return getattr(self.__wrapped, name) + + def __enter__(self, *args, **kwargs): + # special method lookup bypasses __getattr__/__getattribute__, see + # https://stackoverflow.com/questions/12632894/why-doesnt-getattr-work-with-exit + # thus, contextlib magic methods are not proxied via __getattr__ + return self.__wrapped.__enter__(*args, **kwargs) + + def __exit__(self, *args, **kwargs): + return self.__wrapped.__exit__(*args, **kwargs) + + def write(self, text): + self.__convertor.write(text) + + def isatty(self): + stream = self.__wrapped + if 'PYCHARM_HOSTED' in os.environ: + if stream is not None and (stream is sys.__stdout__ or stream is sys.__stderr__): + return True + try: + stream_isatty = stream.isatty + except AttributeError: + return False + else: + return stream_isatty() + + @property + def closed(self): + stream = self.__wrapped + try: + return stream.closed + except AttributeError: + return True + + +class AnsiToWin32(object): + ''' + Implements a 'write()' method which, on Windows, will strip ANSI character + sequences from the text, and if outputting to a tty, will convert them into + win32 function calls. + ''' + ANSI_CSI_RE = re.compile('\001?\033\\[((?:\\d|;)*)([a-zA-Z])\002?') # Control Sequence Introducer + ANSI_OSC_RE = re.compile('\001?\033\\]([^\a]*)(\a)\002?') # Operating System Command + + def __init__(self, wrapped, convert=None, strip=None, autoreset=False): + # The wrapped stream (normally sys.stdout or sys.stderr) + self.wrapped = wrapped + + # should we reset colors to defaults after every .write() + self.autoreset = autoreset + + # create the proxy wrapping our output stream + self.stream = StreamWrapper(wrapped, self) + + on_windows = os.name == 'nt' + # We test if the WinAPI works, because even if we are on Windows + # we may be using a terminal that doesn't support the WinAPI + # (e.g. Cygwin Terminal). In this case it's up to the terminal + # to support the ANSI codes. + conversion_supported = on_windows and winapi_test() + + # should we strip ANSI sequences from our output? + if strip is None: + strip = conversion_supported or (not self.stream.closed and not self.stream.isatty()) + self.strip = strip + + # should we should convert ANSI sequences into win32 calls? + if convert is None: + convert = conversion_supported and not self.stream.closed and self.stream.isatty() + self.convert = convert + + # dict of ansi codes to win32 functions and parameters + self.win32_calls = self.get_win32_calls() + + # are we wrapping stderr? + self.on_stderr = self.wrapped is sys.stderr + + def should_wrap(self): + ''' + True if this class is actually needed. If false, then the output + stream will not be affected, nor will win32 calls be issued, so + wrapping stdout is not actually required. This will generally be + False on non-Windows platforms, unless optional functionality like + autoreset has been requested using kwargs to init() + ''' + return self.convert or self.strip or self.autoreset + + def get_win32_calls(self): + if self.convert and winterm: + return { + AnsiStyle.RESET_ALL: (winterm.reset_all, ), + AnsiStyle.BRIGHT: (winterm.style, WinStyle.BRIGHT), + AnsiStyle.DIM: (winterm.style, WinStyle.NORMAL), + AnsiStyle.NORMAL: (winterm.style, WinStyle.NORMAL), + AnsiFore.BLACK: (winterm.fore, WinColor.BLACK), + AnsiFore.RED: (winterm.fore, WinColor.RED), + AnsiFore.GREEN: (winterm.fore, WinColor.GREEN), + AnsiFore.YELLOW: (winterm.fore, WinColor.YELLOW), + AnsiFore.BLUE: (winterm.fore, WinColor.BLUE), + AnsiFore.MAGENTA: (winterm.fore, WinColor.MAGENTA), + AnsiFore.CYAN: (winterm.fore, WinColor.CYAN), + AnsiFore.WHITE: (winterm.fore, WinColor.GREY), + AnsiFore.RESET: (winterm.fore, ), + AnsiFore.LIGHTBLACK_EX: (winterm.fore, WinColor.BLACK, True), + AnsiFore.LIGHTRED_EX: (winterm.fore, WinColor.RED, True), + AnsiFore.LIGHTGREEN_EX: (winterm.fore, WinColor.GREEN, True), + AnsiFore.LIGHTYELLOW_EX: (winterm.fore, WinColor.YELLOW, True), + AnsiFore.LIGHTBLUE_EX: (winterm.fore, WinColor.BLUE, True), + AnsiFore.LIGHTMAGENTA_EX: (winterm.fore, WinColor.MAGENTA, True), + AnsiFore.LIGHTCYAN_EX: (winterm.fore, WinColor.CYAN, True), + AnsiFore.LIGHTWHITE_EX: (winterm.fore, WinColor.GREY, True), + AnsiBack.BLACK: (winterm.back, WinColor.BLACK), + AnsiBack.RED: (winterm.back, WinColor.RED), + AnsiBack.GREEN: (winterm.back, WinColor.GREEN), + AnsiBack.YELLOW: (winterm.back, WinColor.YELLOW), + AnsiBack.BLUE: (winterm.back, WinColor.BLUE), + AnsiBack.MAGENTA: (winterm.back, WinColor.MAGENTA), + AnsiBack.CYAN: (winterm.back, WinColor.CYAN), + AnsiBack.WHITE: (winterm.back, WinColor.GREY), + AnsiBack.RESET: (winterm.back, ), + AnsiBack.LIGHTBLACK_EX: (winterm.back, WinColor.BLACK, True), + AnsiBack.LIGHTRED_EX: (winterm.back, WinColor.RED, True), + AnsiBack.LIGHTGREEN_EX: (winterm.back, WinColor.GREEN, True), + AnsiBack.LIGHTYELLOW_EX: (winterm.back, WinColor.YELLOW, True), + AnsiBack.LIGHTBLUE_EX: (winterm.back, WinColor.BLUE, True), + AnsiBack.LIGHTMAGENTA_EX: (winterm.back, WinColor.MAGENTA, True), + AnsiBack.LIGHTCYAN_EX: (winterm.back, WinColor.CYAN, True), + AnsiBack.LIGHTWHITE_EX: (winterm.back, WinColor.GREY, True), + } + return dict() + + def write(self, text): + if self.strip or self.convert: + self.write_and_convert(text) + else: + self.wrapped.write(text) + self.wrapped.flush() + if self.autoreset: + self.reset_all() + + + def reset_all(self): + if self.convert: + self.call_win32('m', (0,)) + elif not self.strip and not self.stream.closed: + self.wrapped.write(Style.RESET_ALL) + + + def write_and_convert(self, text): + ''' + Write the given text to our wrapped stream, stripping any ANSI + sequences from the text, and optionally converting them into win32 + calls. + ''' + cursor = 0 + text = self.convert_osc(text) + for match in self.ANSI_CSI_RE.finditer(text): + start, end = match.span() + self.write_plain_text(text, cursor, start) + self.convert_ansi(*match.groups()) + cursor = end + self.write_plain_text(text, cursor, len(text)) + + + def write_plain_text(self, text, start, end): + if start < end: + self.wrapped.write(text[start:end]) + self.wrapped.flush() + + + def convert_ansi(self, paramstring, command): + if self.convert: + params = self.extract_params(command, paramstring) + self.call_win32(command, params) + + + def extract_params(self, command, paramstring): + if command in 'Hf': + params = tuple(int(p) if len(p) != 0 else 1 for p in paramstring.split(';')) + while len(params) < 2: + # defaults: + params = params + (1,) + else: + params = tuple(int(p) for p in paramstring.split(';') if len(p) != 0) + if len(params) == 0: + # defaults: + if command in 'JKm': + params = (0,) + elif command in 'ABCD': + params = (1,) + + return params + + + def call_win32(self, command, params): + if command == 'm': + for param in params: + if param in self.win32_calls: + func_args = self.win32_calls[param] + func = func_args[0] + args = func_args[1:] + kwargs = dict(on_stderr=self.on_stderr) + func(*args, **kwargs) + elif command in 'J': + winterm.erase_screen(params[0], on_stderr=self.on_stderr) + elif command in 'K': + winterm.erase_line(params[0], on_stderr=self.on_stderr) + elif command in 'Hf': # cursor position - absolute + winterm.set_cursor_position(params, on_stderr=self.on_stderr) + elif command in 'ABCD': # cursor position - relative + n = params[0] + # A - up, B - down, C - forward, D - back + x, y = {'A': (0, -n), 'B': (0, n), 'C': (n, 0), 'D': (-n, 0)}[command] + winterm.cursor_adjust(x, y, on_stderr=self.on_stderr) + + + def convert_osc(self, text): + for match in self.ANSI_OSC_RE.finditer(text): + start, end = match.span() + text = text[:start] + text[end:] + paramstring, command = match.groups() + if command == BEL: + if paramstring.count(";") == 1: + params = paramstring.split(";") + # 0 - change title and icon (we will only change title) + # 1 - change icon (we don't support this) + # 2 - change title + if params[0] in '02': + winterm.set_title(params[1]) + return text diff --git a/mosec/colorama/initialise.py b/mosec/colorama/initialise.py new file mode 100644 index 0000000..430d066 --- /dev/null +++ b/mosec/colorama/initialise.py @@ -0,0 +1,80 @@ +# Copyright Jonathan Hartley 2013. BSD 3-Clause license, see LICENSE file. +import atexit +import contextlib +import sys + +from .ansitowin32 import AnsiToWin32 + + +orig_stdout = None +orig_stderr = None + +wrapped_stdout = None +wrapped_stderr = None + +atexit_done = False + + +def reset_all(): + if AnsiToWin32 is not None: # Issue #74: objects might become None at exit + AnsiToWin32(orig_stdout).reset_all() + + +def init(autoreset=False, convert=None, strip=None, wrap=True): + + if not wrap and any([autoreset, convert, strip]): + raise ValueError('wrap=False conflicts with any other arg=True') + + global wrapped_stdout, wrapped_stderr + global orig_stdout, orig_stderr + + orig_stdout = sys.stdout + orig_stderr = sys.stderr + + if sys.stdout is None: + wrapped_stdout = None + else: + sys.stdout = wrapped_stdout = \ + wrap_stream(orig_stdout, convert, strip, autoreset, wrap) + if sys.stderr is None: + wrapped_stderr = None + else: + sys.stderr = wrapped_stderr = \ + wrap_stream(orig_stderr, convert, strip, autoreset, wrap) + + global atexit_done + if not atexit_done: + atexit.register(reset_all) + atexit_done = True + + +def deinit(): + if orig_stdout is not None: + sys.stdout = orig_stdout + if orig_stderr is not None: + sys.stderr = orig_stderr + + +@contextlib.contextmanager +def colorama_text(*args, **kwargs): + init(*args, **kwargs) + try: + yield + finally: + deinit() + + +def reinit(): + if wrapped_stdout is not None: + sys.stdout = wrapped_stdout + if wrapped_stderr is not None: + sys.stderr = wrapped_stderr + + +def wrap_stream(stream, convert, strip, autoreset, wrap): + if wrap: + wrapper = AnsiToWin32(stream, + convert=convert, strip=strip, autoreset=autoreset) + if wrapper.should_wrap(): + stream = wrapper.stream + return stream diff --git a/mosec/colorama/win32.py b/mosec/colorama/win32.py new file mode 100644 index 0000000..c2d8360 --- /dev/null +++ b/mosec/colorama/win32.py @@ -0,0 +1,152 @@ +# Copyright Jonathan Hartley 2013. BSD 3-Clause license, see LICENSE file. + +# from winbase.h +STDOUT = -11 +STDERR = -12 + +try: + import ctypes + from ctypes import LibraryLoader + windll = LibraryLoader(ctypes.WinDLL) + from ctypes import wintypes +except (AttributeError, ImportError): + windll = None + SetConsoleTextAttribute = lambda *_: None + winapi_test = lambda *_: None +else: + from ctypes import byref, Structure, c_char, POINTER + + COORD = wintypes._COORD + + class CONSOLE_SCREEN_BUFFER_INFO(Structure): + """struct in wincon.h.""" + _fields_ = [ + ("dwSize", COORD), + ("dwCursorPosition", COORD), + ("wAttributes", wintypes.WORD), + ("srWindow", wintypes.SMALL_RECT), + ("dwMaximumWindowSize", COORD), + ] + def __str__(self): + return '(%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d)' % ( + self.dwSize.Y, self.dwSize.X + , self.dwCursorPosition.Y, self.dwCursorPosition.X + , self.wAttributes + , self.srWindow.Top, self.srWindow.Left, self.srWindow.Bottom, self.srWindow.Right + , self.dwMaximumWindowSize.Y, self.dwMaximumWindowSize.X + ) + + _GetStdHandle = windll.kernel32.GetStdHandle + _GetStdHandle.argtypes = [ + wintypes.DWORD, + ] + _GetStdHandle.restype = wintypes.HANDLE + + _GetConsoleScreenBufferInfo = windll.kernel32.GetConsoleScreenBufferInfo + _GetConsoleScreenBufferInfo.argtypes = [ + wintypes.HANDLE, + POINTER(CONSOLE_SCREEN_BUFFER_INFO), + ] + _GetConsoleScreenBufferInfo.restype = wintypes.BOOL + + _SetConsoleTextAttribute = windll.kernel32.SetConsoleTextAttribute + _SetConsoleTextAttribute.argtypes = [ + wintypes.HANDLE, + wintypes.WORD, + ] + _SetConsoleTextAttribute.restype = wintypes.BOOL + + _SetConsoleCursorPosition = windll.kernel32.SetConsoleCursorPosition + _SetConsoleCursorPosition.argtypes = [ + wintypes.HANDLE, + COORD, + ] + _SetConsoleCursorPosition.restype = wintypes.BOOL + + _FillConsoleOutputCharacterA = windll.kernel32.FillConsoleOutputCharacterA + _FillConsoleOutputCharacterA.argtypes = [ + wintypes.HANDLE, + c_char, + wintypes.DWORD, + COORD, + POINTER(wintypes.DWORD), + ] + _FillConsoleOutputCharacterA.restype = wintypes.BOOL + + _FillConsoleOutputAttribute = windll.kernel32.FillConsoleOutputAttribute + _FillConsoleOutputAttribute.argtypes = [ + wintypes.HANDLE, + wintypes.WORD, + wintypes.DWORD, + COORD, + POINTER(wintypes.DWORD), + ] + _FillConsoleOutputAttribute.restype = wintypes.BOOL + + _SetConsoleTitleW = windll.kernel32.SetConsoleTitleW + _SetConsoleTitleW.argtypes = [ + wintypes.LPCWSTR + ] + _SetConsoleTitleW.restype = wintypes.BOOL + + def _winapi_test(handle): + csbi = CONSOLE_SCREEN_BUFFER_INFO() + success = _GetConsoleScreenBufferInfo( + handle, byref(csbi)) + return bool(success) + + def winapi_test(): + return any(_winapi_test(h) for h in + (_GetStdHandle(STDOUT), _GetStdHandle(STDERR))) + + def GetConsoleScreenBufferInfo(stream_id=STDOUT): + handle = _GetStdHandle(stream_id) + csbi = CONSOLE_SCREEN_BUFFER_INFO() + success = _GetConsoleScreenBufferInfo( + handle, byref(csbi)) + return csbi + + def SetConsoleTextAttribute(stream_id, attrs): + handle = _GetStdHandle(stream_id) + return _SetConsoleTextAttribute(handle, attrs) + + def SetConsoleCursorPosition(stream_id, position, adjust=True): + position = COORD(*position) + # If the position is out of range, do nothing. + if position.Y <= 0 or position.X <= 0: + return + # Adjust for Windows' SetConsoleCursorPosition: + # 1. being 0-based, while ANSI is 1-based. + # 2. expecting (x,y), while ANSI uses (y,x). + adjusted_position = COORD(position.Y - 1, position.X - 1) + if adjust: + # Adjust for viewport's scroll position + sr = GetConsoleScreenBufferInfo(STDOUT).srWindow + adjusted_position.Y += sr.Top + adjusted_position.X += sr.Left + # Resume normal processing + handle = _GetStdHandle(stream_id) + return _SetConsoleCursorPosition(handle, adjusted_position) + + def FillConsoleOutputCharacter(stream_id, char, length, start): + handle = _GetStdHandle(stream_id) + char = c_char(char.encode()) + length = wintypes.DWORD(length) + num_written = wintypes.DWORD(0) + # Note that this is hard-coded for ANSI (vs wide) bytes. + success = _FillConsoleOutputCharacterA( + handle, char, length, start, byref(num_written)) + return num_written.value + + def FillConsoleOutputAttribute(stream_id, attr, length, start): + ''' FillConsoleOutputAttribute( hConsole, csbi.wAttributes, dwConSize, coordScreen, &cCharsWritten )''' + handle = _GetStdHandle(stream_id) + attribute = wintypes.WORD(attr) + length = wintypes.DWORD(length) + num_written = wintypes.DWORD(0) + # Note that this is hard-coded for ANSI (vs wide) bytes. + return _FillConsoleOutputAttribute( + handle, attribute, length, start, byref(num_written)) + + def SetConsoleTitle(title): + return _SetConsoleTitleW(title) diff --git a/mosec/colorama/winterm.py b/mosec/colorama/winterm.py new file mode 100644 index 0000000..0fdb4ec --- /dev/null +++ b/mosec/colorama/winterm.py @@ -0,0 +1,169 @@ +# Copyright Jonathan Hartley 2013. BSD 3-Clause license, see LICENSE file. +from . import win32 + + +# from wincon.h +class WinColor(object): + BLACK = 0 + BLUE = 1 + GREEN = 2 + CYAN = 3 + RED = 4 + MAGENTA = 5 + YELLOW = 6 + GREY = 7 + +# from wincon.h +class WinStyle(object): + NORMAL = 0x00 # dim text, dim background + BRIGHT = 0x08 # bright text, dim background + BRIGHT_BACKGROUND = 0x80 # dim text, bright background + +class WinTerm(object): + + def __init__(self): + self._default = win32.GetConsoleScreenBufferInfo(win32.STDOUT).wAttributes + self.set_attrs(self._default) + self._default_fore = self._fore + self._default_back = self._back + self._default_style = self._style + # In order to emulate LIGHT_EX in windows, we borrow the BRIGHT style. + # So that LIGHT_EX colors and BRIGHT style do not clobber each other, + # we track them separately, since LIGHT_EX is overwritten by Fore/Back + # and BRIGHT is overwritten by Style codes. + self._light = 0 + + def get_attrs(self): + return self._fore + self._back * 16 + (self._style | self._light) + + def set_attrs(self, value): + self._fore = value & 7 + self._back = (value >> 4) & 7 + self._style = value & (WinStyle.BRIGHT | WinStyle.BRIGHT_BACKGROUND) + + def reset_all(self, on_stderr=None): + self.set_attrs(self._default) + self.set_console(attrs=self._default) + self._light = 0 + + def fore(self, fore=None, light=False, on_stderr=False): + if fore is None: + fore = self._default_fore + self._fore = fore + # Emulate LIGHT_EX with BRIGHT Style + if light: + self._light |= WinStyle.BRIGHT + else: + self._light &= ~WinStyle.BRIGHT + self.set_console(on_stderr=on_stderr) + + def back(self, back=None, light=False, on_stderr=False): + if back is None: + back = self._default_back + self._back = back + # Emulate LIGHT_EX with BRIGHT_BACKGROUND Style + if light: + self._light |= WinStyle.BRIGHT_BACKGROUND + else: + self._light &= ~WinStyle.BRIGHT_BACKGROUND + self.set_console(on_stderr=on_stderr) + + def style(self, style=None, on_stderr=False): + if style is None: + style = self._default_style + self._style = style + self.set_console(on_stderr=on_stderr) + + def set_console(self, attrs=None, on_stderr=False): + if attrs is None: + attrs = self.get_attrs() + handle = win32.STDOUT + if on_stderr: + handle = win32.STDERR + win32.SetConsoleTextAttribute(handle, attrs) + + def get_position(self, handle): + position = win32.GetConsoleScreenBufferInfo(handle).dwCursorPosition + # Because Windows coordinates are 0-based, + # and win32.SetConsoleCursorPosition expects 1-based. + position.X += 1 + position.Y += 1 + return position + + def set_cursor_position(self, position=None, on_stderr=False): + if position is None: + # I'm not currently tracking the position, so there is no default. + # position = self.get_position() + return + handle = win32.STDOUT + if on_stderr: + handle = win32.STDERR + win32.SetConsoleCursorPosition(handle, position) + + def cursor_adjust(self, x, y, on_stderr=False): + handle = win32.STDOUT + if on_stderr: + handle = win32.STDERR + position = self.get_position(handle) + adjusted_position = (position.Y + y, position.X + x) + win32.SetConsoleCursorPosition(handle, adjusted_position, adjust=False) + + def erase_screen(self, mode=0, on_stderr=False): + # 0 should clear from the cursor to the end of the screen. + # 1 should clear from the cursor to the beginning of the screen. + # 2 should clear the entire screen, and move cursor to (1,1) + handle = win32.STDOUT + if on_stderr: + handle = win32.STDERR + csbi = win32.GetConsoleScreenBufferInfo(handle) + # get the number of character cells in the current buffer + cells_in_screen = csbi.dwSize.X * csbi.dwSize.Y + # get number of character cells before current cursor position + cells_before_cursor = csbi.dwSize.X * csbi.dwCursorPosition.Y + csbi.dwCursorPosition.X + if mode == 0: + from_coord = csbi.dwCursorPosition + cells_to_erase = cells_in_screen - cells_before_cursor + elif mode == 1: + from_coord = win32.COORD(0, 0) + cells_to_erase = cells_before_cursor + elif mode == 2: + from_coord = win32.COORD(0, 0) + cells_to_erase = cells_in_screen + else: + # invalid mode + return + # fill the entire screen with blanks + win32.FillConsoleOutputCharacter(handle, ' ', cells_to_erase, from_coord) + # now set the buffer's attributes accordingly + win32.FillConsoleOutputAttribute(handle, self.get_attrs(), cells_to_erase, from_coord) + if mode == 2: + # put the cursor where needed + win32.SetConsoleCursorPosition(handle, (1, 1)) + + def erase_line(self, mode=0, on_stderr=False): + # 0 should clear from the cursor to the end of the line. + # 1 should clear from the cursor to the beginning of the line. + # 2 should clear the entire line. + handle = win32.STDOUT + if on_stderr: + handle = win32.STDERR + csbi = win32.GetConsoleScreenBufferInfo(handle) + if mode == 0: + from_coord = csbi.dwCursorPosition + cells_to_erase = csbi.dwSize.X - csbi.dwCursorPosition.X + elif mode == 1: + from_coord = win32.COORD(0, csbi.dwCursorPosition.Y) + cells_to_erase = csbi.dwCursorPosition.X + elif mode == 2: + from_coord = win32.COORD(0, csbi.dwCursorPosition.Y) + cells_to_erase = csbi.dwSize.X + else: + # invalid mode + return + # fill the entire screen with blanks + win32.FillConsoleOutputCharacter(handle, ' ', cells_to_erase, from_coord) + # now set the buffer's attributes accordingly + win32.FillConsoleOutputAttribute(handle, self.get_attrs(), cells_to_erase, from_coord) + + def set_title(self, title): + win32.SetConsoleTitle(title) diff --git a/mosec/mosec_log_helper.py b/mosec/mosec_log_helper.py new file mode 100644 index 0000000..6a8b15b --- /dev/null +++ b/mosec/mosec_log_helper.py @@ -0,0 +1,30 @@ +import logging +import sys +from mosec.colorama.ansi import Fore, Style + + +class Logger(object): + def __init__(self, name, level=logging.INFO): + self.logger = logging.getLogger(name=name) + # set lowest info critical > error > warning > info > debug + self.logger.setLevel(level) + + self.ch = logging.StreamHandler(sys.stdout) + self.ch.setLevel(level) + self.logger.addHandler(self.ch) + + def debug(self, msg): + self.logger.debug(str(msg)) + + def warn(self, msg): + self.logger.warning(Fore.YELLOW + str(msg) + Style.RESET_ALL) + + def info(self, msg): + self.logger.info(Fore.LIGHTGREEN_EX + str(msg) + Style.RESET_ALL) + + def error(self, msg): + self.logger.error(Fore.LIGHTRED_EX + str(msg) + Style.RESET_ALL) + + def set_log_level(self, level): + self.logger.setLevel(level) + self.ch.setLevel(level) diff --git a/mosec/pip_resolve.py b/mosec/pip_resolve.py new file mode 100644 index 0000000..474a555 --- /dev/null +++ b/mosec/pip_resolve.py @@ -0,0 +1,265 @@ +import logging +import sys +import os +import argparse +import json +import ssl +import urllib.error +import urllib.request +from operator import attrgetter +from mosec import mosec_log_helper +from mosec import setup_file +from mosec import utils +from mosec.requirement_file_parser import get_requirements_list +from mosec.requirement_dist import ReqDist + +try: + import pkg_resources +except ImportError: + # try using the version vendored by pip + try: + import pip._vendor.pkg_resources as pkg_resources + except ImportError: + raise ImportError( + "Could not import pkg_resources; please install setuptools or pip.") + +try: + from collections import OrderedDict +except ImportError: + from ordereddict import OrderedDict + + +log = mosec_log_helper.Logger(name="mosec") + + +def create_deps_tree( + dist_tree, + top_level_requirements, + req_file_path, + allow_missing=False, + only_provenance=False +): + """Create dist dependencies tree + :param dict dist_tree: the installed dists tree + :param list top_level_requirements: list of required dists + :param str req_file_path: path to the dependencies file (e.g. requirements.txt) + :param bool allow_missing: ignore uninstalled dependencies + :param bool only_provenance: only care provenance dependencies + :rtype: dict + """ + DEPENDENCIES = 'dependencies' + VERSION = 'version' + NAME = 'name' + DIR_VERSION = '1.0.0' + FROM = 'from' + + tree = OrderedDict( + sorted( + [(k, sorted(v, key=attrgetter('key'))) for k, v in dist_tree.items()] + , key=lambda kv: kv[0].key + ) + ) + nodes = tree.keys() + key_tree = dict((k.key, v) for k, v in tree.items()) + + top_level_req_lower_names = [p.name.lower() for p in top_level_requirements] + top_level_req_dists = [p for p in nodes if p.key in top_level_req_lower_names] + + def _create_children_recursive(root_package, ancestors): + root_name = root_package[NAME] + if root_name.lower() not in key_tree: + msg = 'Required packages missing: ' + root_name + if allow_missing: + log.error(msg) + return + else: + sys.exit(msg) + + ancestors = ancestors.copy() + ancestors.add(root_name.lower()) + children_dists = key_tree[root_name.lower()] + for child_dist in children_dists: + if child_dist.key in ancestors: + continue + + child_node = _create_tree_node(child_dist, root_package) + _create_children_recursive(child_node, ancestors) + root_package[DEPENDENCIES][child_node[NAME]] = child_node + return root_package + + def _create_root(): + name, version = None, None + if os.path.basename(req_file_path) == 'setup.py': + with open(req_file_path, "r") as setup_py_file: + name, version = setup_file.parse_name_and_version(setup_py_file.read()) + + root = { + NAME: name or os.path.basename(os.path.dirname(os.path.abspath(req_file_path))), + VERSION: version or DIR_VERSION, + DEPENDENCIES: {} + } + root[FROM] = [root[NAME] + '@' + root[VERSION]] + return root + + def _create_tree_node(dist_node, parent): + version = dist_node.version + if isinstance(version, tuple): + version = '.'.join(map(str, version)) + return { + NAME: dist_node.project_name, + VERSION: version, + FROM: parent[FROM] + [dist_node.project_name + '@' + version], + DEPENDENCIES: {} + } + + tree_root = _create_root() + for dist in top_level_req_dists: + tree_node = _create_tree_node(dist, tree_root) + if only_provenance: + tree_root[DEPENDENCIES][tree_node[NAME]] = tree_node + else: + tree_root[DEPENDENCIES][tree_node[NAME]] = _create_children_recursive(tree_node, set([])) + return tree_root + + +def create_dependencies_tree_by_req_file( + requirements_file, + allow_missing=False, + only_provenance=False +): + """Create dist dependencies tree from file + :param str requirements_file: path to the dependencies file (e.g. requirements.txt) + :param bool allow_missing: ignore uninstalled dependencies + :param bool only_provenance: only care provenance dependencies + :rtype: dict + """ + + # get all installed package distribution object list + dists = list(pkg_resources.working_set) + dists_dict = dict((p.key, p) for p in dists) + dists_tree = dict((p, [ReqDist(r, dists_dict.get(r.key)) for r in p.requires()]) for p in dists_dict.values()) + + required = get_requirements_list(requirements_file) + installed = [utils.canonicalize_dist_name(d) for d in dists_dict] + top_level_requirements = [] + missing_package_names = [] + for r in required: + if utils.canonicalize_dist_name(r.name) not in installed: + missing_package_names.append(r.name) + else: + top_level_requirements.append(r) + if missing_package_names: + msg = 'Required packages missing: ' + (', '.join(missing_package_names)) + if allow_missing: + log.error(msg) + else: + sys.exit(msg) + + return create_deps_tree( + dists_tree, top_level_requirements, requirements_file, allow_missing, only_provenance) + + +def render_response(response_json): + + def _print_single_vuln(vuln): + log.error("✗ {} severity vulnerability ({} - {}) found on {}@{}".format( + vuln.get('severity'), + vuln.get('title', ''), + vuln.get('cve', ''), + vuln.get('packageName'), + vuln.get('version')) + ) + if vuln.get('from', None): + from_arr = vuln.get('from') + from_str = "" + for _from in from_arr: + from_str += _from + " > " + from_str = from_str[:-3] + print("- from: {}".format(from_str)) + if vuln.get('target_version', []): + log.info("! Fix version {}".format(vuln.get('target_version'))) + print("") + + if response_json.get('ok', False): + log.info("✓ Tested {} dependencies for known vulnerabilities, no vulnerable paths found." + .format(response_json.get('dependencyCount', 0))) + elif response_json.get('vulnerabilities', None): + vulns = response_json.get('vulnerabilities') + for vuln in vulns: + _print_single_vuln(vuln) + + log.warn("Tested {} dependencies for known vulnerabilities, found {} vulnerable paths." + .format(response_json.get('dependencyCount', 0), len(vulns))) + + +def run(args): + deps_tree = create_dependencies_tree_by_req_file( + args.requirements, + allow_missing=args.allow_missing, + only_provenance=args.only_provenance, + ) + + deps_tree['severityLevel'] = args.level + deps_tree['type'] = 'pip' + deps_tree['language'] = 'python' + + log.debug(json.dumps(deps_tree, indent=2)) + + ctx = ssl.create_default_context() + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + + req = urllib.request.Request( + method='POST', + url=args.endpoint, + headers={ + 'Content-Type': 'application/json' + }, + data=bytes(json.dumps(deps_tree).encode('utf-8')) + ) + try: + response = urllib.request.urlopen(req, timeout=15, context=ctx) + response_json = json.loads(response.read().decode('utf-8')) + render_response(response_json) + + if not response_json.get('ok', False): + return 1 + except urllib.error.HTTPError as e: + raise Exception("Network Error: {}".format(e)) + except json.JSONDecodeError as e: + raise Exception("API return data format error.") + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("requirements", + help="依赖文件 (requirements.txt 或 Pipfile)") + parser.add_argument("--endpoint", + action="store", + required=True, + help="上报API") + parser.add_argument("--allow-missing", + action="store_true", + help="忽略未安装的依赖") + parser.add_argument("--only-provenance", + action="store_true", + help="仅检查直接依赖") + parser.add_argument("--level", + action="store", + default="High", + help="威胁等级 [High|Medium|Low]. default: High") + parser.add_argument("--no-except", + action="store_true", + default=False, + help="发现漏洞不抛出异常") + parser.add_argument("--debug", + action="store_true", + default=False) + args = parser.parse_args() + + if args.debug: + log.set_log_level(logging.DEBUG) + + status = run(args) + if status == 1 and not args.no_except: + raise BaseException("Found Vulnerable!") diff --git a/mosec/pipfile.py b/mosec/pipfile.py new file mode 100644 index 0000000..f550644 --- /dev/null +++ b/mosec/pipfile.py @@ -0,0 +1,63 @@ +"""Simplistic parsing of Pipfile dependency files + +This only extracts a small subset of the information present in a Pipfile, +as needed for the purposes of this library. +""" +from .utils import is_string +from .pytoml import loads as pytoml_loads + + +class PipfileRequirement(object): + def __init__(self, name): + self.name = name + + self.editable = False + self.vcs = None + self.vcs_uri = None + self.version = None + self.markers = None + self.provenance = None # a tuple of (file name, line) + + @classmethod + def from_dict(cls, name, requirement_dict, pos_in_toml): + req = cls(name) + + req.version = requirement_dict.get('version') + req.editable = requirement_dict.get('editable', False) + for vcs in ['git', 'hg', 'svn', 'bzr']: + if vcs in requirement_dict: + req.vcs = vcs + req.vcs_uri = requirement_dict[vcs] + break + req.markers = requirement_dict.get('markers') + # proper file name to be injected into provenance by the calling code + req.provenance = ('Pipfile', pos_in_toml[0], pos_in_toml[0]) + + return req + + +def val_with_pos(kind, text, value, pos): + return (value, pos) + + +def parse(file_contents): + data = pytoml_loads(file_contents, translate=val_with_pos) + + sections = ['packages', 'dev-packages'] + res = dict.fromkeys(sections) + for section in sections: + if section not in data: + continue + + section_data = data[section] + + res[section] = [ + PipfileRequirement.from_dict( + name, + value if not is_string(value) else {'version': value}, + pos, + ) + for name, (value, pos) in sorted(section_data.items()) + ] + + return res diff --git a/mosec/pytoml/LICENSE b/mosec/pytoml/LICENSE new file mode 100644 index 0000000..9739fc6 --- /dev/null +++ b/mosec/pytoml/LICENSE @@ -0,0 +1,16 @@ +No-notice MIT License + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/mosec/pytoml/README.txt b/mosec/pytoml/README.txt new file mode 100644 index 0000000..d9f4f95 --- /dev/null +++ b/mosec/pytoml/README.txt @@ -0,0 +1,9 @@ +This is pytoml v0.1.16, taken from the avakar/pytoml GitHub repo. + +See: https://github.com/avakar/pytoml/releases/tag/v0.1.16 + +It is bundled out of necessity due to constraints of the +Snyk CLI plugin architecture. + +Also, it contains a Snyk-specific patch: the user-configurable "translate" +function now also takes a fourth argument, "pos". diff --git a/mosec/pytoml/__init__.py b/mosec/pytoml/__init__.py new file mode 100644 index 0000000..8dc7315 --- /dev/null +++ b/mosec/pytoml/__init__.py @@ -0,0 +1,3 @@ +from .core import TomlError +from .parser import load, loads +from .writer import dump, dumps diff --git a/mosec/pytoml/core.py b/mosec/pytoml/core.py new file mode 100644 index 0000000..c182734 --- /dev/null +++ b/mosec/pytoml/core.py @@ -0,0 +1,13 @@ +class TomlError(RuntimeError): + def __init__(self, message, line, col, filename): + RuntimeError.__init__(self, message, line, col, filename) + self.message = message + self.line = line + self.col = col + self.filename = filename + + def __str__(self): + return '{}({}, {}): {}'.format(self.filename, self.line, self.col, self.message) + + def __repr__(self): + return 'TomlError({!r}, {!r}, {!r}, {!r})'.format(self.message, self.line, self.col, self.filename) diff --git a/mosec/pytoml/parser.py b/mosec/pytoml/parser.py new file mode 100644 index 0000000..98671ad --- /dev/null +++ b/mosec/pytoml/parser.py @@ -0,0 +1,374 @@ +import string, re, sys, datetime +from .core import TomlError + +if sys.version_info[0] == 2: + _chr = unichr +else: + _chr = chr + +def load(fin, translate=lambda t, x, v, pos: v, object_pairs_hook=dict): + return loads(fin.read(), translate=translate, object_pairs_hook=object_pairs_hook, filename=getattr(fin, 'name', repr(fin))) + +def loads(s, filename='', translate=lambda t, x, v, pos: v, object_pairs_hook=dict): + if isinstance(s, bytes): + s = s.decode('utf-8') + + s = s.replace('\r\n', '\n') + + root = object_pairs_hook() + tables = object_pairs_hook() + scope = root + + src = _Source(s, filename=filename) + ast = _p_toml(src, object_pairs_hook=object_pairs_hook) + + def error(msg): + raise TomlError(msg, pos[0], pos[1], filename) + + def process_value(v, object_pairs_hook): + kind, text, value, pos = v + if kind == 'str' and value.startswith('\n'): + value = value[1:] + if kind == 'array': + if value and any(k != value[0][0] for k, t, v, p in value[1:]): + error('array-type-mismatch') + value = [process_value(item, object_pairs_hook=object_pairs_hook) for item in value] + elif kind == 'table': + value = object_pairs_hook([(k, process_value(value[k], object_pairs_hook=object_pairs_hook)) for k in value]) + return translate(kind, text, value, pos) + + for kind, value, pos in ast: + if kind == 'kv': + k, v = value + if k in scope: + error('duplicate_keys. Key "{0}" was used more than once.'.format(k)) + scope[k] = process_value(v, object_pairs_hook=object_pairs_hook) + else: + is_table_array = (kind == 'table_array') + cur = tables + for name in value[:-1]: + if isinstance(cur.get(name), list): + d, cur = cur[name][-1] + else: + d, cur = cur.setdefault(name, (None, object_pairs_hook())) + + scope = object_pairs_hook() + name = value[-1] + if name not in cur: + if is_table_array: + cur[name] = [(scope, object_pairs_hook())] + else: + cur[name] = (scope, object_pairs_hook()) + elif isinstance(cur[name], list): + if not is_table_array: + error('table_type_mismatch') + cur[name].append((scope, object_pairs_hook())) + else: + if is_table_array: + error('table_type_mismatch') + old_scope, next_table = cur[name] + if old_scope is not None: + error('duplicate_tables') + cur[name] = (scope, next_table) + + def merge_tables(scope, tables): + if scope is None: + scope = object_pairs_hook() + for k in tables: + if k in scope: + error('key_table_conflict') + v = tables[k] + if isinstance(v, list): + scope[k] = [merge_tables(sc, tbl) for sc, tbl in v] + else: + scope[k] = merge_tables(v[0], v[1]) + return scope + + return merge_tables(root, tables) + +class _Source: + def __init__(self, s, filename=None): + self.s = s + self._pos = (1, 1) + self._last = None + self._filename = filename + self.backtrack_stack = [] + + def last(self): + return self._last + + def pos(self): + return self._pos + + def fail(self): + return self._expect(None) + + def consume_dot(self): + if self.s: + self._last = self.s[0] + self.s = self[1:] + self._advance(self._last) + return self._last + return None + + def expect_dot(self): + return self._expect(self.consume_dot()) + + def consume_eof(self): + if not self.s: + self._last = '' + return True + return False + + def expect_eof(self): + return self._expect(self.consume_eof()) + + def consume(self, s): + if self.s.startswith(s): + self.s = self.s[len(s):] + self._last = s + self._advance(s) + return True + return False + + def expect(self, s): + return self._expect(self.consume(s)) + + def consume_re(self, re): + m = re.match(self.s) + if m: + self.s = self.s[len(m.group(0)):] + self._last = m + self._advance(m.group(0)) + return m + return None + + def expect_re(self, re): + return self._expect(self.consume_re(re)) + + def __enter__(self): + self.backtrack_stack.append((self.s, self._pos)) + + def __exit__(self, type, value, traceback): + if type is None: + self.backtrack_stack.pop() + else: + self.s, self._pos = self.backtrack_stack.pop() + return type == TomlError + + def commit(self): + self.backtrack_stack[-1] = (self.s, self._pos) + + def _expect(self, r): + if not r: + raise TomlError('msg', self._pos[0], self._pos[1], self._filename) + return r + + def _advance(self, s): + suffix_pos = s.rfind('\n') + if suffix_pos == -1: + self._pos = (self._pos[0], self._pos[1] + len(s)) + else: + self._pos = (self._pos[0] + s.count('\n'), len(s) - suffix_pos) + +_ews_re = re.compile(r'(?:[ \t]|#[^\n]*\n|#[^\n]*\Z|\n)*') +def _p_ews(s): + s.expect_re(_ews_re) + +_ws_re = re.compile(r'[ \t]*') +def _p_ws(s): + s.expect_re(_ws_re) + +_escapes = { 'b': '\b', 'n': '\n', 'r': '\r', 't': '\t', '"': '"', '\'': '\'', + '\\': '\\', '/': '/', 'f': '\f' } + +_basicstr_re = re.compile(r'[^"\\\000-\037]*') +_short_uni_re = re.compile(r'u([0-9a-fA-F]{4})') +_long_uni_re = re.compile(r'U([0-9a-fA-F]{8})') +_escapes_re = re.compile('[bnrt"\'\\\\/f]') +_newline_esc_re = re.compile('\n[ \t\n]*') +def _p_basicstr_content(s, content=_basicstr_re): + res = [] + while True: + res.append(s.expect_re(content).group(0)) + if not s.consume('\\'): + break + if s.consume_re(_newline_esc_re): + pass + elif s.consume_re(_short_uni_re) or s.consume_re(_long_uni_re): + res.append(_chr(int(s.last().group(1), 16))) + else: + s.expect_re(_escapes_re) + res.append(_escapes[s.last().group(0)]) + return ''.join(res) + +_key_re = re.compile(r'[0-9a-zA-Z-_]+') +def _p_key(s): + with s: + s.expect('"') + r = _p_basicstr_content(s, _basicstr_re) + s.expect('"') + return r + if s.consume('\''): + if s.consume('\'\''): + r = s.expect_re(_litstr_ml_re).group(0) + s.expect('\'\'\'') + else: + r = s.expect_re(_litstr_re).group(0) + s.expect('\'') + return r + return s.expect_re(_key_re).group(0) + +_float_re = re.compile(r'[+-]?(?:0|[1-9](?:_?\d)*)(?:\.\d(?:_?\d)*)?(?:[eE][+-]?(?:\d(?:_?\d)*))?') +_datetime_re = re.compile(r'(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(\.\d+)?(?:Z|([+-]\d{2}):(\d{2}))') + +_basicstr_ml_re = re.compile(r'(?:(?:|"|"")[^"\\\000-\011\013-\037])*') +_litstr_re = re.compile(r"[^'\000-\037]*") +_litstr_ml_re = re.compile(r"(?:(?:|'|'')(?:[^'\000-\011\013-\037]))*") +def _p_value(s, object_pairs_hook): + pos = s.pos() + + if s.consume('true'): + return 'bool', s.last(), True, pos + if s.consume('false'): + return 'bool', s.last(), False, pos + + if s.consume('"'): + if s.consume('""'): + r = _p_basicstr_content(s, _basicstr_ml_re) + s.expect('"""') + else: + r = _p_basicstr_content(s, _basicstr_re) + s.expect('"') + return 'str', r, r, pos + + if s.consume('\''): + if s.consume('\'\''): + r = s.expect_re(_litstr_ml_re).group(0) + s.expect('\'\'\'') + else: + r = s.expect_re(_litstr_re).group(0) + s.expect('\'') + return 'str', r, r, pos + + if s.consume_re(_datetime_re): + m = s.last() + s0 = m.group(0) + r = map(int, m.groups()[:6]) + if m.group(7): + micro = float(m.group(7)) + else: + micro = 0 + + if m.group(8): + g = int(m.group(8), 10) * 60 + int(m.group(9), 10) + tz = _TimeZone(datetime.timedelta(0, g * 60)) + else: + tz = _TimeZone(datetime.timedelta(0, 0)) + + y, m, d, H, M, S = r + dt = datetime.datetime(y, m, d, H, M, S, int(micro * 1000000), tz) + return 'datetime', s0, dt, pos + + if s.consume_re(_float_re): + m = s.last().group(0) + r = m.replace('_','') + if '.' in m or 'e' in m or 'E' in m: + return 'float', m, float(r), pos + else: + return 'int', m, int(r, 10), pos + + if s.consume('['): + items = [] + with s: + while True: + _p_ews(s) + items.append(_p_value(s, object_pairs_hook=object_pairs_hook)) + s.commit() + _p_ews(s) + s.expect(',') + s.commit() + _p_ews(s) + s.expect(']') + return 'array', None, items, pos + + if s.consume('{'): + _p_ws(s) + items = object_pairs_hook() + if not s.consume('}'): + k = _p_key(s) + _p_ws(s) + s.expect('=') + _p_ws(s) + items[k] = _p_value(s, object_pairs_hook=object_pairs_hook) + _p_ws(s) + while s.consume(','): + _p_ws(s) + k = _p_key(s) + _p_ws(s) + s.expect('=') + _p_ws(s) + items[k] = _p_value(s, object_pairs_hook=object_pairs_hook) + _p_ws(s) + s.expect('}') + return 'table', None, items, pos + + s.fail() + +def _p_stmt(s, object_pairs_hook): + pos = s.pos() + if s.consume( '['): + is_array = s.consume('[') + _p_ws(s) + keys = [_p_key(s)] + _p_ws(s) + while s.consume('.'): + _p_ws(s) + keys.append(_p_key(s)) + _p_ws(s) + s.expect(']') + if is_array: + s.expect(']') + return 'table_array' if is_array else 'table', keys, pos + + key = _p_key(s) + _p_ws(s) + s.expect('=') + _p_ws(s) + value = _p_value(s, object_pairs_hook=object_pairs_hook) + return 'kv', (key, value), pos + +_stmtsep_re = re.compile(r'(?:[ \t]*(?:#[^\n]*)?\n)+[ \t]*') +def _p_toml(s, object_pairs_hook): + stmts = [] + _p_ews(s) + with s: + stmts.append(_p_stmt(s, object_pairs_hook=object_pairs_hook)) + while True: + s.commit() + s.expect_re(_stmtsep_re) + stmts.append(_p_stmt(s, object_pairs_hook=object_pairs_hook)) + _p_ews(s) + s.expect_eof() + return stmts + +class _TimeZone(datetime.tzinfo): + def __init__(self, offset): + self._offset = offset + + def utcoffset(self, dt): + return self._offset + + def dst(self, dt): + return None + + def tzname(self, dt): + m = self._offset.total_seconds() // 60 + if m < 0: + res = '-' + m = -m + else: + res = '+' + h = m // 60 + m = m - h * 60 + return '{}{:.02}{:.02}'.format(res, h, m) diff --git a/mosec/pytoml/writer.py b/mosec/pytoml/writer.py new file mode 100644 index 0000000..6eaf5d7 --- /dev/null +++ b/mosec/pytoml/writer.py @@ -0,0 +1,127 @@ +from __future__ import unicode_literals +import io, datetime, math, sys + +if sys.version_info[0] == 3: + long = int + unicode = str + + +def dumps(obj, sort_keys=False): + fout = io.StringIO() + dump(obj, fout, sort_keys=sort_keys) + return fout.getvalue() + + +_escapes = {'\n': 'n', '\r': 'r', '\\': '\\', '\t': 't', '\b': 'b', '\f': 'f', '"': '"'} + + +def _escape_string(s): + res = [] + start = 0 + + def flush(): + if start != i: + res.append(s[start:i]) + return i + 1 + + i = 0 + while i < len(s): + c = s[i] + if c in '"\\\n\r\t\b\f': + start = flush() + res.append('\\' + _escapes[c]) + elif ord(c) < 0x20: + start = flush() + res.append('\\u%04x' % ord(c)) + i += 1 + + flush() + return '"' + ''.join(res) + '"' + + +def _escape_id(s): + if any(not c.isalnum() and c not in '-_' for c in s): + return _escape_string(s) + return s + + +def _format_list(v): + return '[{0}]'.format(', '.join(_format_value(obj) for obj in v)) + +# Formula from: +# https://docs.python.org/2/library/datetime.html#datetime.timedelta.total_seconds +# Once support for py26 is dropped, this can be replaced by td.total_seconds() +def _total_seconds(td): + return ((td.microseconds + + (td.seconds + td.days * 24 * 3600) * 10**6) / 10.0**6) + +def _format_value(v): + if isinstance(v, bool): + return 'true' if v else 'false' + if isinstance(v, int) or isinstance(v, long): + return unicode(v) + if isinstance(v, float): + if math.isnan(v) or math.isinf(v): + raise ValueError("{0} is not a valid TOML value".format(v)) + else: + return repr(v) + elif isinstance(v, unicode) or isinstance(v, bytes): + return _escape_string(v) + elif isinstance(v, datetime.datetime): + offs = v.utcoffset() + offs = _total_seconds(offs) // 60 if offs is not None else 0 + + if offs == 0: + suffix = 'Z' + else: + if offs > 0: + suffix = '+' + else: + suffix = '-' + offs = -offs + suffix = '{0}{1:.02}{2:.02}'.format(suffix, offs // 60, offs % 60) + + if v.microsecond: + return v.strftime('%Y-%m-%dT%H:%M:%S.%f') + suffix + else: + return v.strftime('%Y-%m-%dT%H:%M:%S') + suffix + elif isinstance(v, list): + return _format_list(v) + else: + raise RuntimeError(v) + + +def dump(obj, fout, sort_keys=False): + tables = [((), obj, False)] + + while tables: + name, table, is_array = tables.pop() + if name: + section_name = '.'.join(_escape_id(c) for c in name) + if is_array: + fout.write('[[{0}]]\n'.format(section_name)) + else: + fout.write('[{0}]\n'.format(section_name)) + + table_keys = sorted(table.keys()) if sort_keys else table.keys() + new_tables = [] + has_kv = False + for k in table_keys: + v = table[k] + if isinstance(v, dict): + new_tables.append((name + (k,), v, False)) + elif isinstance(v, list) and v and all(isinstance(o, dict) for o in v): + new_tables.extend((name + (k,), d, True) for d in v) + elif v is None: + # based on mojombo's comment: https://github.com/toml-lang/toml/issues/146#issuecomment-25019344 + fout.write( + '#{} = null # To use: uncomment and replace null with value\n'.format(_escape_id(k))) + has_kv = True + else: + fout.write('{0} = {1}\n'.format(_escape_id(k), _format_value(v))) + has_kv = True + + tables.extend(reversed(new_tables)) + + if (name or has_kv) and tables: + fout.write('\n') diff --git a/mosec/requirement_dist.py b/mosec/requirement_dist.py new file mode 100644 index 0000000..80ecb99 --- /dev/null +++ b/mosec/requirement_dist.py @@ -0,0 +1,15 @@ +from pkg_resources import Requirement +from mosec import utils + + +class ReqDist(Requirement): + + def __init__(self, req, dist=None): + super(ReqDist, self).__init__(str(req)) + self.dist = dist + + @property + def version(self): + if not self.dist: + return utils.guess_version(self.key) + return self.dist.version diff --git a/mosec/requirement_file_parser.py b/mosec/requirement_file_parser.py new file mode 100644 index 0000000..be08c77 --- /dev/null +++ b/mosec/requirement_file_parser.py @@ -0,0 +1,102 @@ +import re +import sys +import os +from operator import le, lt, gt, ge, eq, ne +from mosec import pipfile +from mosec import requirements +from mosec import setup_file +from pkg_resources._vendor.packaging.version import parse as version_parser + +PYTHON_MARKER_REGEX = re.compile(r'python_version\s*(?P==|<=|=>|>|<)\s*[\'"](?P.+?)[\'"]') +SYSTEM_MARKER_REGEX = re.compile(r'sys_platform\s*==\s*[\'"](.+)[\'"]') + + +def satisfies_python_version(parsed_operator, py_version_str): + operator_func = { + ">": gt, + "==": eq, + "<": lt, + "<=": le, + ">=": ge, + '!=': ne, + }[parsed_operator] + system_py_version = version_parser("{}.{}.{}".format(sys.version_info[0], sys.version_info[1], sys.version_info[2])) + required_py_version = version_parser(py_version_str) + return operator_func(system_py_version, required_py_version) + + +def get_markers_text(requirement): + if isinstance(requirement, pipfile.PipfileRequirement): + return requirement.markers + return requirement.line + + +def matches_python_version(requirement): + """Filter out requirements that should not be installed + in this Python version. + See: https://www.python.org/dev/peps/pep-0508/#environment-markers + """ + markers_text = get_markers_text(requirement) + if not (markers_text and re.match(".*;.*python_version", markers_text)): + return True + + cond_text = markers_text.split(";", 1)[1] + + # Gloss over the 'and' case and return true on the first matching python version + + for sub_exp in re.split("\s*(?:and|or)\s*", cond_text): + match = PYTHON_MARKER_REGEX.search(sub_exp) + + if match: + match_dict = match.groupdict() + + if len(match_dict) == 2 and satisfies_python_version( + match_dict['operator'], + match_dict['python_version'] + ): + return True + + return False + + +def matches_environment(requirement): + """Filter out requirements that should not be installed + in this environment. Only sys_platform is inspected right now. + This should be expanded to include other environment markers. + See: https://www.python.org/dev/peps/pep-0508/#environment-markers + """ + sys_platform = sys.platform.lower() + markers_text = get_markers_text(requirement) + if markers_text and 'sys_platform' in markers_text: + match = SYSTEM_MARKER_REGEX.findall(markers_text) + if len(match) > 0: + return match[0].lower() == sys_platform + return True + + +def is_testable(requirement): + return not requirement.editable and requirement.vcs is None + + +def get_requirements_list(requirements_file_path): + if os.path.basename(requirements_file_path) == 'Pipfile': + with open(requirements_file_path, 'r', encoding='utf-8') as f: + requirements_data = f.read() + parsed_reqs = pipfile.parse(requirements_data) + req_list = list(parsed_reqs.get('packages', [])) + elif os.path.basename(requirements_file_path) == 'setup.py': + with open(requirements_file_path, 'r') as f: + setup_py_file_content = f.read() + requirements_data = setup_file.parse_requirements(setup_py_file_content) + req_list = list(requirements.parse(requirements_data)) + else: + # assume this is a requirements.txt formatted file + # Note: requirements.txt files are unicode and can be in any encoding. + with open(requirements_file_path, 'r') as f: + req_list = list(requirements.parse(f)) + + req_list = filter(matches_environment, req_list) + req_list = filter(is_testable, req_list) + req_list = filter(matches_python_version, req_list) + req_list = [r for r in req_list if r.name] + return req_list diff --git a/mosec/requirements/LICENSE.rst b/mosec/requirements/LICENSE.rst new file mode 100644 index 0000000..20a9573 --- /dev/null +++ b/mosec/requirements/LICENSE.rst @@ -0,0 +1,29 @@ +License +======= + +Requirements Parser is licensed under the BSD license. + +Copyright (c) 2012 - 2013, David Fischer + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +- Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY David Fischer ''AS IS'' AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL David Fischer BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + diff --git a/mosec/requirements/README.rst b/mosec/requirements/README.rst new file mode 100644 index 0000000..60c940e --- /dev/null +++ b/mosec/requirements/README.rst @@ -0,0 +1,57 @@ +Requirements Parser +=================== + + +.. image:: https://travis-ci.org/davidfischer/requirements-parser.svg?branch=master + :target: https://travis-ci.org/davidfischer/requirements-parser +.. image:: https://coveralls.io/repos/github/davidfischer/requirements-parser/badge.svg?branch=master + :target: https://coveralls.io/github/davidfischer/requirements-parser?branch=master +.. image:: http://readthedocs.org/projects/requirements-parser/badge/?version=latest + :target: http://requirements-parser.readthedocs.io/en/latest/?badge=latest + :alt: Documentation Status + +This is a small Python module for parsing Pip_ requirement files. + +The goal is to parse everything in the `Pip requirement file format`_ spec. + +.. _Pip: http://www.pip-installer.org/ +.. _Pip requirement file format: https://pip.pypa.io/en/stable/reference/pip_install/#requirements-file-format + + +Installation +============ + +:: + + pip install requirements-parser + + +Examples +======== + +Requirements parser can parse a file-like object or a text string. + +.. code-block:: python + + >>> import requirements + >>> with open('requirements.txt', 'r') as fd: + ... for req in requirements.parse(fd): + ... print(req.name, req.specs) + Django [('>=', '1.11'), ('<', '1.12')] + six [('==', '1.10.0')] + +It can handle most if not all of the options in requirement files that do +not involve traversing the local filesystem. These include: + +* editables (`-e git+https://github.com/toastdriven/pyelasticsearch.git`) +* version control URIs +* egg hashes and subdirectories (`#egg=django-haystack&subdirectory=setup`) +* extras (`DocParser[PDF]`) +* URLs + + +Documentation +============= + +For more details and examples, the documentation is available at: +http://requirements-parser.readthedocs.io. diff --git a/mosec/requirements/__init__.py b/mosec/requirements/__init__.py new file mode 100644 index 0000000..36349d2 --- /dev/null +++ b/mosec/requirements/__init__.py @@ -0,0 +1,22 @@ +from .parser import parse # noqa + +_MAJOR = 0 +_MINOR = 2 +_PATCH = 0 + + +def version_tuple(): + ''' + Returns a 3-tuple of ints that represent the version + ''' + return (_MAJOR, _MINOR, _PATCH) + + +def version(): + ''' + Returns a string representation of the version + ''' + return '%d.%d.%d' % (version_tuple()) + + +__version__ = version() diff --git a/mosec/requirements/fragment.py b/mosec/requirements/fragment.py new file mode 100644 index 0000000..a89f7a6 --- /dev/null +++ b/mosec/requirements/fragment.py @@ -0,0 +1,44 @@ +import re + +# Copied from pip +# https://github.com/pypa/pip/blob/281eb61b09d87765d7c2b92f6982b3fe76ccb0af/pip/index.py#L947 +HASH_ALGORITHMS = set(['sha1', 'sha224', 'sha384', 'sha256', 'sha512', 'md5']) + +extras_require_search = re.compile( + r'(?P.+)\[(?P[^\]]+)\]').search + + +def parse_fragment(fragment_string): + """Takes a fragment string nd returns a dict of the components""" + fragment_string = fragment_string.lstrip('#') + + try: + return dict( + key_value_string.split('=') + for key_value_string in fragment_string.split('&') + ) + except ValueError: + raise ValueError( + 'Invalid fragment string {fragment_string}'.format( + fragment_string=fragment_string + ) + ) + + +def get_hash_info(d): + """Returns the first matching hashlib name and value from a dict""" + for key in d.keys(): + if key.lower() in HASH_ALGORITHMS: + return key, d[key] + + return None, None + + +def parse_extras_require(egg): + if egg is not None: + match = extras_require_search(egg) + if match is not None: + name = match.group('name') + extras = match.group('extras') + return name, [extra.strip() for extra in extras.split(',')] + return egg, [] diff --git a/mosec/requirements/parser.py b/mosec/requirements/parser.py new file mode 100644 index 0000000..cfbbaea --- /dev/null +++ b/mosec/requirements/parser.py @@ -0,0 +1,50 @@ +import os +import warnings + +from .requirement import Requirement + + +def parse(reqstr): + """ + Parse a requirements file into a list of Requirements + + See: pip/req.py:parse_requirements() + + :param reqstr: a string or file like object containing requirements + :returns: a *generator* of Requirement objects + """ + filename = getattr(reqstr, 'name', None) + try: + # Python 2.x compatibility + if not isinstance(reqstr, basestring): + reqstr = reqstr.read() + except NameError: + # Python 3.x only + if not isinstance(reqstr, str): + reqstr = reqstr.read() + + for line in reqstr.splitlines(): + line = line.strip() + if line == '': + continue + elif not line or line.startswith('#'): + # comments are lines that start with # only + continue + elif line.startswith('-r') or line.startswith('--requirement'): + _, new_filename = line.split() + new_file_path = os.path.join(os.path.dirname(filename or '.'), + new_filename) + with open(new_file_path) as f: + for requirement in parse(f): + yield requirement + elif line.startswith('-f') or line.startswith('--find-links') or \ + line.startswith('-i') or line.startswith('--index-url') or \ + line.startswith('--extra-index-url') or \ + line.startswith('--no-index'): + warnings.warn('Private repos not supported. Skipping.') + continue + elif line.startswith('-Z') or line.startswith('--always-unzip'): + warnings.warn('Unused option --always-unzip. Skipping.') + continue + else: + yield Requirement.parse(line, filename) diff --git a/mosec/requirements/requirement.py b/mosec/requirements/requirement.py new file mode 100644 index 0000000..8c0d1ec --- /dev/null +++ b/mosec/requirements/requirement.py @@ -0,0 +1,258 @@ +from __future__ import unicode_literals +import re +import os +import warnings +from pkg_resources import Requirement as Req + +from .fragment import get_hash_info, parse_fragment, parse_extras_require +from .vcs import VCS, VCS_SCHEMES + + +URI_REGEX = re.compile( + r'^(?Phttps?|file|ftps?)://(?P[^#]+)' + r'(#(?P\S+))?' +) + +VCS_REGEX = re.compile( + r'^(?P{0})://'.format(r'|'.join( + [scheme.replace('+', r'\+') for scheme in VCS_SCHEMES])) + + r'((?P[^/@]+)@)?' + r'(?P[^#@]+)' + r'(@(?P[^#]+))?' + r'(#(?P\S+))?' +) + +# This matches just about everyting +LOCAL_REGEX = re.compile( + r'^((?Pfile)://)?' + r'(?P[^#]+)' + + r'(#(?P\S+))?' +) + +NAME_EQ_REGEX = re.compile(r'name=(?:\'|")(\S+)(?:\'|")') + + +class Requirement(object): + """ + Represents a single requirement + + Typically instances of this class are created with ``Requirement.parse``. + For local file requirements, there's no verification that the file + exists. This class attempts to be *dict-like*. + + See: http://www.pip-installer.org/en/latest/logic.html + + **Members**: + + * ``line`` - the actual requirement line being parsed + * ``editable`` - a boolean whether this requirement is "editable" + * ``local_file`` - a boolean whether this requirement is a local file/path + * ``specifier`` - a boolean whether this requirement used a requirement + specifier (eg. "django>=1.5" or "requirements") + * ``vcs`` - a string specifying the version control system + * ``revision`` - a version control system specifier + * ``name`` - the name of the requirement + * ``uri`` - the URI if this requirement was specified by URI + * ``subdirectory`` - the subdirectory fragment of the URI + * ``path`` - the local path to the requirement + * ``hash_name`` - the type of hashing algorithm indicated in the line + * ``hash`` - the hash value indicated by the requirement line + * ``extras`` - a list of extras for this requirement + (eg. "mymodule[extra1, extra2]") + * ``specs`` - a list of specs for this requirement + (eg. "mymodule>1.5,<1.6" => [('>', '1.5'), ('<', '1.6')]) + * ``provenance`` - a tuple of (file_name, start_line, end_line), line numbers are 1-based + """ + + def __init__(self, line): + # Do not call this private method + self.line = line + self.editable = False + self.local_file = False + self.specifier = False + self.vcs = None + self.name = None + self.subdirectory = None + self.uri = None + self.path = None + self.revision = None + self.hash_name = None + self.hash = None + self.extras = [] + self.specs = [] + self.provenance = None + + def __repr__(self): + return ''.format(self.line) + + def __getitem__(self, key): + return getattr(self, key) + + def __eq__(self, other): + return all([ + self.name == other.name, + set(self.specs) == set(other.specs), + self.editable == other.editable, + self.specifier == other.specifier, + self.revision == other.revision, + self.hash_name == other.hash_name, + self.hash == other.hash, + set(self.extras) == set(other.extras), + ]) + + def __ne__(self, other): + return not self == other + + def keys(self): + return self.__dict__.keys() + + @classmethod + def parse_editable(cls, line): + """ + Parses a Requirement from an "editable" requirement which is either + a local project path or a VCS project URI. + + See: pip/req.py:from_editable() + + :param line: an "editable" requirement + :returns: a Requirement instance for the given line + :raises: ValueError on an invalid requirement + """ + + req = cls('-e {0}'.format(line)) + req.editable = True + + if ' #' in line: + line = line[:line.find(' #')] + + vcs_match = VCS_REGEX.match(line) + local_match = LOCAL_REGEX.match(line) + + if vcs_match is not None: + groups = vcs_match.groupdict() + if groups.get('login'): + req.uri = '{scheme}://{login}@{path}'.format(**groups) + else: + req.uri = '{scheme}://{path}'.format(**groups) + req.revision = groups['revision'] + if groups['fragment']: + fragment = parse_fragment(groups['fragment']) + egg = fragment.get('egg') + req.name, req.extras = parse_extras_require(egg) + req.hash_name, req.hash = get_hash_info(fragment) + req.subdirectory = fragment.get('subdirectory') + for vcs in VCS: + if req.uri.startswith(vcs): + req.vcs = vcs + else: + assert local_match is not None, 'This should match everything' + groups = local_match.groupdict() + req.local_file = True + if groups['fragment']: + fragment = parse_fragment(groups['fragment']) + egg = fragment.get('egg') + req.name, req.extras = parse_extras_require(egg) + req.hash_name, req.hash = get_hash_info(fragment) + req.subdirectory = fragment.get('subdirectory') + req.path = groups['path'] + + return req + + @classmethod + def parse_line(cls, line, filename=''): + """ + Parses a Requirement from a non-editable requirement. + + See: pip/req.py:from_line() + + :param line: a "non-editable" requirement + :param filename: (optional) where/to/requirements.txt + :returns: a Requirement instance for the given line + :raises: ValueError on an invalid requirement + """ + + req = cls(line) + + vcs_match = VCS_REGEX.match(line) + uri_match = URI_REGEX.match(line) + local_match = LOCAL_REGEX.match(line) + + if vcs_match is not None: + groups = vcs_match.groupdict() + if groups.get('login'): + req.uri = '{scheme}://{login}@{path}'.format(**groups) + else: + req.uri = '{scheme}://{path}'.format(**groups) + req.revision = groups['revision'] + if groups['fragment']: + fragment = parse_fragment(groups['fragment']) + egg = fragment.get('egg') + req.name, req.extras = parse_extras_require(egg) + req.hash_name, req.hash = get_hash_info(fragment) + req.subdirectory = fragment.get('subdirectory') + for vcs in VCS: + if req.uri.startswith(vcs): + req.vcs = vcs + elif uri_match is not None: + groups = uri_match.groupdict() + req.uri = '{scheme}://{path}'.format(**groups) + if groups['fragment']: + fragment = parse_fragment(groups['fragment']) + egg = fragment.get('egg') + req.name, req.extras = parse_extras_require(egg) + req.hash_name, req.hash = get_hash_info(fragment) + req.subdirectory = fragment.get('subdirectory') + if groups['scheme'] == 'file': + req.local_file = True + elif '#egg=' in line: + # Assume a local file match + assert local_match is not None, 'This should match everything' + groups = local_match.groupdict() + req.local_file = True + if groups['fragment']: + fragment = parse_fragment(groups['fragment']) + egg = fragment.get('egg') + name, extras = parse_extras_require(egg) + req.name = fragment.get('egg') + req.hash_name, req.hash = get_hash_info(fragment) + req.subdirectory = fragment.get('subdirectory') + req.path = groups['path'] + elif os.path.isfile(os.path.join(os.path.dirname(filename or '.'), line, "setup.py")): + setup_file = open(os.path.join(os.path.dirname(filename or '.'), line, "setup.py"), "r") + setup_content = setup_file.read() + setup_file.close() + name_search = NAME_EQ_REGEX.search(setup_content) + if name_search: + req.name = name_search.group(1) + req.local_file = True + else: + # This is a requirement specifier. + # Delegate to pkg_resources and hope for the best + req.specifier = True + try: + pkg_req = Req.parse(line) + req.name = pkg_req.unsafe_name + req.extras = list(pkg_req.extras) + req.specs = pkg_req.specs + except Exception as e: + warnings.warn("unkonwn line: {}".format(line)) + return req + + @classmethod + def parse(cls, line, filename=''): + """ + Parses a Requirement from a line of a requirement file. + + :param line: a line of a requirement file + :param filename: (optional) where/to/requirements.txt + :returns: a Requirement instance for the given line + :raises: ValueError on an invalid requirement + """ + + if line.startswith('-e') or line.startswith('--editable'): + # Editable installs are either a local project path + # or a VCS project URI + return cls.parse_editable( + re.sub(r'^(-e|--editable=?)\s*', '', line)) + + return cls.parse_line(line, filename) diff --git a/mosec/requirements/vcs.py b/mosec/requirements/vcs.py new file mode 100644 index 0000000..f5317b2 --- /dev/null +++ b/mosec/requirements/vcs.py @@ -0,0 +1,30 @@ +from __future__ import unicode_literals + +VCS = [ + 'git', + 'hg', + 'svn', + 'bzr', +] + +VCS_SCHEMES = [ + 'git', + 'git+https', + 'git+ssh', + 'git+git', + 'hg+http', + 'hg+https', + 'hg+static-http', + 'hg+ssh', + 'svn', + 'svn+svn', + 'svn+http', + 'svn+https', + 'svn+ssh', + 'bzr+http', + 'bzr+https', + 'bzr+ssh', + 'bzr+sftp', + 'bzr+ftp', + 'bzr+lp', +] diff --git a/mosec/setup_file.py b/mosec/setup_file.py new file mode 100644 index 0000000..eaea648 --- /dev/null +++ b/mosec/setup_file.py @@ -0,0 +1,40 @@ +import distutils.core +import re +import setuptools + + +def parse(setup_py_content): + """Parse a setup.py file and extract the arguments passed to the setup method""" + + # Make the setup method return the arguments that are passed to it + def _save_passed_args(**kwargs): + return kwargs + + distutils.core.setup = _save_passed_args + setuptools.setup = _save_passed_args + setup_py_content = setup_py_content.replace("setup(", "passed_arguments = setup(") + + # Fetch the arguments that were passed to the setup.py + exec(setup_py_content, globals()) + return globals()["passed_arguments"] + + +def parse_name_and_version(setup_py_content): + """Extract the name and version from a setup.py file""" + passed_arguments = parse(setup_py_content) + return passed_arguments.get("name"), passed_arguments.get("version") + + +def get_provenance(setup_py_content): + """Provenance for a setup.py file is the index of the line that contains `install_requires`""" + for index, line in enumerate(setup_py_content.splitlines()): + if re.search(r"(packages|install_requires)\s*=", line): + return index + 1 + return -1 + + +def parse_requirements(setup_py_content): + """Extract the dependencies from a setup.py file""" + passed_arguments = parse(setup_py_content) + requirements = passed_arguments.get("install_requires", passed_arguments.get("packages", [])) + return "\n".join(requirements) diff --git a/mosec/utils.py b/mosec/utils.py new file mode 100644 index 0000000..2951e48 --- /dev/null +++ b/mosec/utils.py @@ -0,0 +1,34 @@ +import sys +import re +from importlib import import_module + + +def canonicalize_dist_name(name): + # https://packaging.python.org/guides/distributing-packages-using-setuptools/#name + name = name.lower().replace('-', '.').replace('_', '.') + name = re.sub(r'\.+', '.', name) + return name + + +def guess_version(pkg_key, default='?'): + """Guess the version of a pkg when pip doesn't provide it + :param str pkg_key: key of the package + :param str default: default version to return if unable to find + :returns: version + :rtype: string + """ + try: + m = import_module(pkg_key) + except ImportError: + return default + else: + return getattr(m, '__version__', default) + + +def is_string(obj): + """Check whether an object is a string""" + if sys.version_info < (3,): + # Python 2.x only + return isinstance(obj, basestring) + else: + return isinstance(obj, str) diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..9c54d13 --- /dev/null +++ b/setup.py @@ -0,0 +1,32 @@ +from setuptools import setup, find_packages + +setup( + name="mosec-pip-plugin", + version="1.1.0", + author="retanoj", + author_email="mmsrc@immomo.com", + description="用于检测python项目的第三方依赖组件是否存在安全漏洞", + license="Apache License 2.0", + url="", + packages=find_packages(), + include_package_data=True, + classifiers=[ + "Environment :: Web Environment", + 'Intended Audience :: Developers', + 'Natural Language :: Chinese', + 'Operating System :: MacOS', + 'Operating System :: Unix', + 'Topic :: Security', + 'Topic :: Software Development :: Libraries :: Python Modules', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + ], + install_requires=[ + ], + entry_points={ + 'console_scripts': ['mosec=mosec.pip_resolve:main'], + }, + zip_safe=True, +)