diff --git a/experimental/rules_python_external/.bazelignore b/experimental/rules_python_external/.bazelignore new file mode 100644 index 0000000000..90c978b36e --- /dev/null +++ b/experimental/rules_python_external/.bazelignore @@ -0,0 +1 @@ +example/ diff --git a/experimental/rules_python_external/.bazelrc b/experimental/rules_python_external/.bazelrc new file mode 100644 index 0000000000..fa4bb2e33f --- /dev/null +++ b/experimental/rules_python_external/.bazelrc @@ -0,0 +1,2 @@ +build --aspects @mypy_integration//:mypy.bzl%mypy_aspect +build --output_groups=+mypy diff --git a/experimental/rules_python_external/.gitattributes b/experimental/rules_python_external/.gitattributes new file mode 100644 index 0000000000..0bad51cbd1 --- /dev/null +++ b/experimental/rules_python_external/.gitattributes @@ -0,0 +1 @@ +example/* linguist-vendored diff --git a/experimental/rules_python_external/.github/workflows/continuous-integration.yml b/experimental/rules_python_external/.github/workflows/continuous-integration.yml new file mode 100644 index 0000000000..01c4cdba23 --- /dev/null +++ b/experimental/rules_python_external/.github/workflows/continuous-integration.yml @@ -0,0 +1,24 @@ +name: CI + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + test: + runs-on: ubuntu-latest + + steps: + # Checks-out the repository under $GITHUB_WORKSPACE, so the job can access it + - uses: actions/checkout@v2 + + - name: Setup Bazel + uses: abhinavsingh/setup-bazel@v3 + with: + # Bazel version to install e.g. 1.2.1, 2.0.0, ... + version: 2.0.0 # optional, default is 2.0.0 + + - name: Run tests + run: bazel test //... diff --git a/experimental/rules_python_external/.gitignore b/experimental/rules_python_external/.gitignore new file mode 100644 index 0000000000..74570cb0c1 --- /dev/null +++ b/experimental/rules_python_external/.gitignore @@ -0,0 +1,135 @@ +# Intellij +.ijwb/ +.idea/ + +# Bazel +bazel-* + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ diff --git a/experimental/rules_python_external/BUILD b/experimental/rules_python_external/BUILD new file mode 100644 index 0000000000..e69de29bb2 diff --git a/experimental/rules_python_external/LICENSE b/experimental/rules_python_external/LICENSE new file mode 100644 index 0000000000..261eeb9e9f --- /dev/null +++ b/experimental/rules_python_external/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/experimental/rules_python_external/README.md b/experimental/rules_python_external/README.md new file mode 100644 index 0000000000..c21dda017f --- /dev/null +++ b/experimental/rules_python_external/README.md @@ -0,0 +1,125 @@ +# rules_python_external ![](https://github.com/dillon-giacoppo/rules_python_external/workflows/CI/badge.svg) + +Bazel rules to transitively fetch and install Python dependencies from a requirements.txt file. + +## Features + +The rules address most of the top packaging issues in [`bazelbuild/rules_python`](https://github.com/bazelbuild/rules_python). This means the rules support common packages such +as [`tensorflow`](https://pypi.org/project/tensorflow/) and [`google.cloud`](https://github.com/googleapis/google-cloud-python) natively. + +* Transitive dependency resolution: + [#35](https://github.com/bazelbuild/rules_python/issues/35), + [#102](https://github.com/bazelbuild/rules_python/issues/102) +* Minimal runtime dependencies: + [#184](https://github.com/bazelbuild/rules_python/issues/184) +* Support for [spreading purelibs](https://www.python.org/dev/peps/pep-0491/#installing-a-wheel-distribution-1-0-py32-none-any-whl): + [#71](https://github.com/bazelbuild/rules_python/issues/71) +* Support for [namespace packages](https://packaging.python.org/guides/packaging-namespace-packages/): + [#14](https://github.com/bazelbuild/rules_python/issues/14), + [#55](https://github.com/bazelbuild/rules_python/issues/55), + [#65](https://github.com/bazelbuild/rules_python/issues/65), + [#93](https://github.com/bazelbuild/rules_python/issues/93), + [#189](https://github.com/bazelbuild/rules_python/issues/189) +* Fetches pip packages only for building Python targets: + [#96](https://github.com/bazelbuild/rules_python/issues/96) +* Reproducible builds: + [#154](https://github.com/bazelbuild/rules_python/issues/154), + [#176](https://github.com/bazelbuild/rules_python/issues/176) + +## Usage + +#### Prerequisites + +The rules support Python >= 3.5 (the oldest [maintained release](https://devguide.python.org/#status-of-python-branches)). + +#### Setup `WORKSPACE` + +```python +rules_python_external_version = "{COMMIT_SHA}" + +http_archive( + name = "rules_python_external", + sha256 = "", # Fill in with correct sha256 of your COMMIT_SHA version + strip_prefix = "rules_python_external-{version}".format(version = rules_python_external_version), + url = "https://github.com/dillon-giacoppo/rules_python_external/archive/v{version}.zip".format(version = rules_python_external_version), +) + +# Install the rule dependencies +load("@rules_python_external//:repositories.bzl", "rules_python_external_dependencies") +rules_python_external_dependencies() + +load("@rules_python_external//:defs.bzl", "pip_install") +pip_install( + name = "py_deps", + requirements = "//:requirements.txt", + # (Optional) You can provide a python interpreter (by path): + python_interpreter = "/usr/bin/python3.8", + # (Optional) Alternatively you can provide an in-build python interpreter, that is available as a Bazel target. + # This overrides `python_interpreter`. + # Note: You need to set up the interpreter target beforehand (not shown here). Please see the `example` folder for further details. + #python_interpreter_target = "@python_interpreter//:python_bin", +) +``` + +#### Example `BUILD` file. + +```python +load("@py_deps//:requirements.bzl", "requirement") + +py_binary( + name = "main", + srcs = ["main.py"], + deps = [ + requirement("boto3"), + ], +) +``` + +Note that above you do not need to add transitively required packages to `deps = [ ... ]` + +#### Setup `requirements.txt` + +While `rules_python_external` **does not** require a _transitively-closed_ `requirements.txt` file, it is recommended. +But if you want to just have top-level packages listed, that also will work. + +Transitively-closed requirements specs are very tedious to produce and maintain manually. To automate the process we +recommend [`pip-compile` from `jazzband/pip-tools`](https://github.com/jazzband/pip-tools#example-usage-for-pip-compile). + +For example, `pip-compile` takes a `requirements.in` like this: + +``` +boto3~=1.9.227 +botocore~=1.12.247 +click~=7.0 +``` + +`pip-compile` 'compiles' it so you get a transitively-closed `requirements.txt` like this, which should be passed to +`pip_install` below: + +``` +boto3==1.9.253 +botocore==1.12.253 +click==7.0 +docutils==0.15.2 # via botocore +jmespath==0.9.4 # via boto3, botocore +python-dateutil==2.8.1 # via botocore +s3transfer==0.2.1 # via boto3 +six==1.14.0 # via python-dateutil +urllib3==1.25.8 # via botocore +``` + +### Demo + +You can find a demo in the [example/](./example) directory. + +## Development + +### Testing + +`bazel test //...` + +## Adopters + +Here's a (non-exhaustive) list of companies that use `rules_python_external` in production. Don't see yours? [You can add it in a PR](https://github.com/dillon-giacoppo/rules_python_external/edit/master/README.md)! + +* [Canva](https://www.canva.com/) diff --git a/experimental/rules_python_external/WORKSPACE b/experimental/rules_python_external/WORKSPACE new file mode 100644 index 0000000000..ad38285c72 --- /dev/null +++ b/experimental/rules_python_external/WORKSPACE @@ -0,0 +1,43 @@ +workspace(name = "rules_python_external") + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +http_archive( + name = "rules_python", + sha256 = "d2865e2ce23ee217aaa408ddaa024ca472114a6f250b46159d27de05530c75e3", + strip_prefix = "rules_python-7b222cfdb4e59b9fd2a609e1fbb233e94fdcde7c", + url = "https://github.com/bazelbuild/rules_python/archive/7b222cfdb4e59b9fd2a609e1fbb233e94fdcde7c.tar.gz", +) + +load("@rules_python//python:repositories.bzl", "py_repositories") +py_repositories() + +load("//:repositories.bzl", "rules_python_external_dependencies") +rules_python_external_dependencies() + +mypy_integration_version = "0.0.7" # latest @ Feb 10th 2020 + +http_archive( + name = "mypy_integration", + sha256 = "bf7ecd386740328f96c343dca095a63b93df7f86f8d3e1e2e6ff46e400880077", # for 0.0.7 + strip_prefix = "bazel-mypy-integration-{version}".format(version = mypy_integration_version), + url = "https://github.com/thundergolfer/bazel-mypy-integration/archive/{version}.zip".format( + version = mypy_integration_version + ), +) + +load( + "@mypy_integration//repositories:repositories.bzl", + mypy_integration_repositories = "repositories", +) + +mypy_integration_repositories() + +load("@mypy_integration//:config.bzl", "mypy_configuration") +mypy_configuration("//tools/typing:mypy.ini") + +load("@mypy_integration//repositories:deps.bzl", mypy_integration_deps = "deps") +mypy_integration_deps("//tools/typing:mypy_version.txt") + +load("@mypy_integration//repositories:pip_repositories.bzl", "pip_deps") +pip_deps() diff --git a/experimental/rules_python_external/defs.bzl b/experimental/rules_python_external/defs.bzl new file mode 100644 index 0000000000..bbac671a30 --- /dev/null +++ b/experimental/rules_python_external/defs.bzl @@ -0,0 +1,108 @@ +load("//:repositories.bzl", "all_requirements") + +DEFAULT_REPOSITORY_NAME = "pip" + +def _pip_repository_impl(rctx): + python_interpreter = rctx.attr.python_interpreter + if rctx.attr.python_interpreter_target != None: + target = rctx.attr.python_interpreter_target + python_interpreter = rctx.path(target) + else: + if "/" not in python_interpreter: + python_interpreter = rctx.which(python_interpreter) + if not python_interpreter: + fail("python interpreter not found") + + rctx.file("BUILD", "") + + # Get the root directory of these rules + rules_root = rctx.path(Label("//:BUILD")).dirname + thirdparty_roots = [ + # Includes all the external dependencies from repositories.bzl + rctx.path(Label("@" + repo + "//:BUILD.bazel")).dirname + for repo in all_requirements + ] + separator = ":" if not "windows" in rctx.os.name.lower() else ";" + pypath = separator.join([str(p) for p in [rules_root] + thirdparty_roots]) + + args = [ + python_interpreter, + "-m", + "extract_wheels", + "--requirements", + rctx.path(rctx.attr.requirements), + "--repo", + "@%s" % rctx.attr.name, + ] + + if rctx.attr.extra_pip_args: + args += [ + "--extra_pip_args", + struct(args = rctx.attr.extra_pip_args).to_json(), + ] + + if rctx.attr.pip_data_exclude: + args += [ + "--pip_data_exclude", + struct(exclude = rctx.attr.pip_data_exclude).to_json(), + ] + + if rctx.attr.enable_implicit_namespace_pkgs: + args += [ + "--enable_implicit_namespace_pkgs" + ] + + result = rctx.execute( + args, + environment = { + # Manually construct the PYTHONPATH since we cannot use the toolchain here + "PYTHONPATH": pypath, + }, + timeout = rctx.attr.timeout, + quiet = rctx.attr.quiet, + ) + if result.return_code: + fail("rules_python_external failed: %s (%s)" % (result.stdout, result.stderr)) + + return + +pip_repository = repository_rule( + attrs = { + "requirements": attr.label(allow_single_file = True, mandatory = True), + "wheel_env": attr.string_dict(), + "python_interpreter": attr.string(default = "python3"), + "python_interpreter_target": attr.label(allow_single_file = True, doc = """ +If you are using a custom python interpreter built by another repository rule, +use this attribute to specify its BUILD target. This allows pip_repository to invoke +pip using the same interpreter as your toolchain. If set, takes precedence over +python_interpreter. +"""), + # 600 is documented as default here: https://docs.bazel.build/versions/master/skylark/lib/repository_ctx.html#execute + "timeout": attr.int(default = 600), + "quiet": attr.bool(default = True), + "extra_pip_args": attr.string_list( + doc = "Extra arguments to pass on to pip. Must not contain spaces.", + ), + "pip_data_exclude": attr.string_list( + doc = "Additional data exclusion parameters to add to the pip packages BUILD file.", + ), + "enable_implicit_namespace_pkgs": attr.bool( + default = False, + doc = """ +If true, disables conversion of native namespace packages into pkg-util style namespace packages. When set all py_binary +and py_test targets must specify either `legacy_create_init=False` or the global Bazel option +`--incompatible_default_to_explicit_init_py` to prevent `__init__.py` being automatically generated in every directory. + +This option is required to support some packages which cannot handle the conversion to pkg-util style. + """, + ), + }, + implementation = _pip_repository_impl, +) + +def pip_install(requirements, name = DEFAULT_REPOSITORY_NAME, **kwargs): + pip_repository( + name = name, + requirements = requirements, + **kwargs + ) diff --git a/experimental/rules_python_external/example/BUILD b/experimental/rules_python_external/example/BUILD new file mode 100644 index 0000000000..64a3718ccf --- /dev/null +++ b/experimental/rules_python_external/example/BUILD @@ -0,0 +1,35 @@ +load("@pip//:requirements.bzl", "requirement") + +# Toolchain setup, this is optional. +# Demonstrate that we can use the same python interpreter for the toolchain and executing pip in pip install (see WORKSPACE). +# +#load("@rules_python//python:defs.bzl", "py_runtime_pair") +# +#py_runtime( +# name = "python3_runtime", +# files = ["@python_interpreter//:files"], +# interpreter = "@python_interpreter//:python_bin", +# python_version = "PY3", +# visibility = ["//visibility:public"], +#) +# +#py_runtime_pair( +# name = "my_py_runtime_pair", +# py2_runtime = None, +# py3_runtime = ":python3_runtime", +#) +# +#toolchain( +# name = "my_py_toolchain", +# toolchain = ":my_py_runtime_pair", +# toolchain_type = "@bazel_tools//tools/python:toolchain_type", +#) +# End of toolchain setup. + +py_binary( + name = "main", + srcs = ["main.py"], + deps = [ + requirement("boto3"), + ], +) diff --git a/experimental/rules_python_external/example/WORKSPACE b/experimental/rules_python_external/example/WORKSPACE new file mode 100644 index 0000000000..639308a5fd --- /dev/null +++ b/experimental/rules_python_external/example/WORKSPACE @@ -0,0 +1,90 @@ +workspace(name = "example_repo") + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +http_archive( + name = "rules_python", + url = "https://github.com/bazelbuild/rules_python/releases/download/0.0.2/rules_python-0.0.2.tar.gz", + strip_prefix = "rules_python-0.0.2", + sha256 = "b5668cde8bb6e3515057ef465a35ad712214962f0b3a314e551204266c7be90c", +) + +load("@rules_python//python:repositories.bzl", "py_repositories") + +py_repositories() + +local_repository( + name = "rules_python_external", + path = "../", +) + +load("@rules_python_external//:repositories.bzl", "rules_python_external_dependencies") + +rules_python_external_dependencies() + +load("@rules_python_external//:defs.bzl", "pip_install") + +pip_install( + # (Optional) You can provide extra parameters to pip. + # Here, make pip output verbose (this is usable with `quiet = False`). + #extra_pip_args = ["-v"], + + # (Optional) You can exclude custom elements in the data section of the generated BUILD files for pip packages. + # Exclude directories with spaces in their names in this example (avoids build errors if there are such directories). + #pip_data_exclude = ["**/* */**"], + + # (Optional) You can provide a python_interpreter (path) or a python_interpreter_target (a Bazel target, that + # acts as an executable). The latter can be anything that could be used as Python interpreter. E.g.: + # 1. Python interpreter that you compile in the build file (as above in @python_interpreter). + # 2. Pre-compiled python interpreter included with http_archive + # 3. Wrapper script, like in the autodetecting python toolchain. + #python_interpreter_target = "@python_interpreter//:python_bin", + + # (Optional) You can set quiet to False if you want to see pip output. + #quiet = False, + + # Uses the default repository name "pip" + requirements = "//:requirements.txt", +) + +# You could optionally use an in-build, compiled python interpreter as a toolchain, +# and also use it to execute pip. +# +# Special logic for building python interpreter with OpenSSL from homebrew. +# See https://devguide.python.org/setup/#macos-and-os-x +#_py_configure = """ +#if [[ "$OSTYPE" == "darwin"* ]]; then +# ./configure --prefix=$(pwd)/bazel_install --with-openssl=$(brew --prefix openssl) +#else +# ./configure --prefix=$(pwd)/bazel_install +#fi +#""" +# +# NOTE: you need to have the SSL headers installed to build with openssl support (and use HTTPS). +# E.g. on Ubuntu: `sudo apt install libssl-dev` +#http_archive( +# name = "python_interpreter", +# build_file_content = """ +#exports_files(["python_bin"]) +#filegroup( +# name = "files", +# srcs = glob(["bazel_install/**"], exclude = ["**/* *"]), +# visibility = ["//visibility:public"], +#) +#""", +# patch_cmds = [ +# "mkdir $(pwd)/bazel_install", +# _py_configure, +# "make", +# "make install", +# "ln -s bazel_install/bin/python3 python_bin", +# ], +# sha256 = "dfab5ec723c218082fe3d5d7ae17ecbdebffa9a1aea4d64aa3a2ecdd2e795864", +# strip_prefix = "Python-3.8.3", +# urls = ["https://www.python.org/ftp/python/3.8.3/Python-3.8.3.tar.xz"], +#) + +# Optional: +# Register the toolchain with the same python interpreter we used for pip in pip_install(). +#register_toolchains("//:my_py_toolchain") +# End of in-build Python interpreter setup. diff --git a/experimental/rules_python_external/example/main.py b/experimental/rules_python_external/example/main.py new file mode 100644 index 0000000000..e5c690b621 --- /dev/null +++ b/experimental/rules_python_external/example/main.py @@ -0,0 +1,4 @@ +import boto3 + +if __name__ == "__main__": + pass diff --git a/experimental/rules_python_external/example/requirements.txt b/experimental/rules_python_external/example/requirements.txt new file mode 100644 index 0000000000..30ddf823b8 --- /dev/null +++ b/experimental/rules_python_external/example/requirements.txt @@ -0,0 +1 @@ +boto3 diff --git a/experimental/rules_python_external/extract_wheels/BUILD b/experimental/rules_python_external/extract_wheels/BUILD new file mode 100644 index 0000000000..afd2f45ce3 --- /dev/null +++ b/experimental/rules_python_external/extract_wheels/BUILD @@ -0,0 +1,8 @@ +load("//:repositories.bzl", "all_requirements") + +py_binary( + name = "extract_wheels", + srcs = ["__init__.py", "__main__.py"], + main = "__main__.py", + deps = ["//extract_wheels/lib"], +) diff --git a/experimental/rules_python_external/extract_wheels/__init__.py b/experimental/rules_python_external/extract_wheels/__init__.py new file mode 100644 index 0000000000..8184dac6aa --- /dev/null +++ b/experimental/rules_python_external/extract_wheels/__init__.py @@ -0,0 +1,110 @@ +"""extract_wheels + +extract_wheels resolves and fetches artifacts transitively from the Python Package Index (PyPI) based on a +requirements.txt. It generates the required BUILD files to consume these packages as Python libraries. + +Under the hood, it depends on the `pip wheel` command to do resolution, download, and compilation into wheels. +""" +import argparse +import glob +import os +import subprocess +import sys +import json + +from extract_wheels.lib import bazel, requirements + + +def configure_reproducible_wheels() -> None: + """Modifies the environment to make wheel building reproducible. + + Wheels created from sdists are not reproducible by default. We can however workaround this by + patching in some configuration with environment variables. + """ + + # wheel, by default, enables debug symbols in GCC. This incidentally captures the build path in the .so file + # We can override this behavior by disabling debug symbols entirely. + # https://github.com/pypa/pip/issues/6505 + if "CFLAGS" in os.environ: + os.environ["CFLAGS"] += " -g0" + else: + os.environ["CFLAGS"] = "-g0" + + # set SOURCE_DATE_EPOCH to 1980 so that we can use python wheels + # https://github.com/NixOS/nixpkgs/blob/master/doc/languages-frameworks/python.section.md#python-setuppy-bdist_wheel-cannot-create-whl + if "SOURCE_DATE_EPOCH" not in os.environ: + os.environ["SOURCE_DATE_EPOCH"] = "315532800" + + # Python wheel metadata files can be unstable. + # See https://bitbucket.org/pypa/wheel/pull-requests/74/make-the-output-of-metadata-files/diff + if "PYTHONHASHSEED" not in os.environ: + os.environ["PYTHONHASHSEED"] = "0" + + +def main() -> None: + """Main program. + + Exits zero on successful program termination, non-zero otherwise. + """ + + configure_reproducible_wheels() + + parser = argparse.ArgumentParser( + description="Resolve and fetch artifacts transitively from PyPI" + ) + parser.add_argument( + "--requirements", + action="store", + required=True, + help="Path to requirements.txt from where to install dependencies", + ) + parser.add_argument( + "--repo", + action="store", + required=True, + help="The external repo name to install dependencies. In the format '@{REPO_NAME}'", + ) + parser.add_argument( + "--extra_pip_args", action="store", help="Extra arguments to pass down to pip.", + ) + parser.add_argument( + "--pip_data_exclude", + action="store", + help="Additional data exclusion parameters to add to the pip packages BUILD file.", + ) + parser.add_argument( + "--enable_implicit_namespace_pkgs", + action="store_true", + help="Disables conversion of implicit namespace packages into pkg-util style packages.", + ) + args = parser.parse_args() + + pip_args = [sys.executable, "-m", "pip", "wheel", "-r", args.requirements] + if args.extra_pip_args: + pip_args += json.loads(args.extra_pip_args)["args"] + + # Assumes any errors are logged by pip so do nothing. This command will fail if pip fails + subprocess.run(pip_args, check=True) + + extras = requirements.parse_extras(args.requirements) + + if args.pip_data_exclude: + pip_data_exclude = json.loads(args.pip_data_exclude)["exclude"] + else: + pip_data_exclude = [] + + targets = [ + '"%s%s"' + % ( + args.repo, + bazel.extract_wheel( + whl, extras, pip_data_exclude, args.enable_implicit_namespace_pkgs + ), + ) + for whl in glob.glob("*.whl") + ] + + with open("requirements.bzl", "w") as requirement_file: + requirement_file.write( + bazel.generate_requirements_file_contents(args.repo, targets) + ) diff --git a/experimental/rules_python_external/extract_wheels/__main__.py b/experimental/rules_python_external/extract_wheels/__main__.py new file mode 100644 index 0000000000..939e8b98c1 --- /dev/null +++ b/experimental/rules_python_external/extract_wheels/__main__.py @@ -0,0 +1,5 @@ +"""Main entry point.""" +import extract_wheels + +if __name__ == "__main__": + extract_wheels.main() diff --git a/experimental/rules_python_external/extract_wheels/lib/BUILD b/experimental/rules_python_external/extract_wheels/lib/BUILD new file mode 100644 index 0000000000..ef6e0ab3e4 --- /dev/null +++ b/experimental/rules_python_external/extract_wheels/lib/BUILD @@ -0,0 +1,41 @@ +load("//:repositories.bzl", "requirement") + +py_library( + name = "lib", + visibility = ["//extract_wheels:__subpackages__"], + srcs = [ + "bazel.py", + "namespace_pkgs.py", + "purelib.py", + "requirements.py", + "wheel.py", + ], + deps = [ + requirement("pkginfo"), + requirement("setuptools"), + ], +) + +py_test( + name = "namespace_pkgs_test", + size = "small", + srcs = [ + "namespace_pkgs_test.py", + ], + tags = ["unit"], + deps = [ + ":lib", + ], +) + +py_test( + name = "requirements_test", + size = "small", + srcs = [ + "requirements_test.py", + ], + tags = ["unit"], + deps = [ + ":lib", + ], +) diff --git a/experimental/rules_python_external/extract_wheels/lib/__init__.py b/experimental/rules_python_external/extract_wheels/lib/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/experimental/rules_python_external/extract_wheels/lib/bazel.py b/experimental/rules_python_external/extract_wheels/lib/bazel.py new file mode 100644 index 0000000000..acda4c29fa --- /dev/null +++ b/experimental/rules_python_external/extract_wheels/lib/bazel.py @@ -0,0 +1,166 @@ +"""Utility functions to manipulate Bazel files""" +import os +import textwrap +import json +from typing import Iterable, List, Dict, Set + +from extract_wheels.lib import namespace_pkgs, wheel, purelib + + +def generate_build_file_contents( + name: str, dependencies: List[str], pip_data_exclude: List[str] +) -> str: + """Generate a BUILD file for an unzipped Wheel + + Args: + name: the target name of the py_library + dependencies: a list of Bazel labels pointing to dependencies of the library + + Returns: + A complete BUILD file as a string + + We allow for empty Python sources as for Wheels containing only compiled C code + there may be no Python sources whatsoever (e.g. packages written in Cython: like `pymssql`). + """ + + data_exclude = ["**/*.py", "**/* *", "BUILD", "WORKSPACE"] + pip_data_exclude + + return textwrap.dedent( + """\ + package(default_visibility = ["//visibility:public"]) + + load("@rules_python//python:defs.bzl", "py_library") + + py_library( + name = "{name}", + srcs = glob(["**/*.py"], allow_empty = True), + data = glob(["**/*"], exclude={data_exclude}), + # This makes this directory a top-level in the python import + # search path for anything that depends on this. + imports = ["."], + deps = [{dependencies}], + ) + """.format( + name=name, + dependencies=",".join(dependencies), + data_exclude=json.dumps(data_exclude), + ) + ) + + +def generate_requirements_file_contents(repo_name: str, targets: Iterable[str]) -> str: + """Generate a requirements.bzl file for a given pip repository + + The file allows converting the PyPI name to a bazel label. Additionally, it adds a function which can glob all the + installed dependencies. This is provided for legacy reasons and can be considered deprecated. + + Args: + repo_name: the name of the pip repository + targets: a list of Bazel labels pointing to all the generated targets + + Returns: + A complete requirements.bzl file as a string + """ + + return textwrap.dedent( + """\ + # Deprecated. This will be removed in a future release + all_requirements = [{requirement_labels}] + + def requirement(name): + name_key = name.replace("-", "_").replace(".", "_").lower() + return "{repo}//pypi__" + name_key + """.format( + repo=repo_name, requirement_labels=",".join(sorted(targets)) + ) + ) + + +def sanitise_name(name: str) -> str: + """Sanitises the name to be compatible with Bazel labels. + + There are certain requirements around Bazel labels that we need to consider. From the Bazel docs: + + Package names must be composed entirely of characters drawn from the set A-Z, a–z, 0–9, '/', '-', '.', and '_', + and cannot start with a slash. + + Due to restrictions on Bazel labels we also cannot allow hyphens. See + https://github.com/bazelbuild/bazel/issues/6841 + + Further, rules-python automatically adds the repository root to the PYTHONPATH, meaning a package that has the same + name as a module is picked up. We workaround this by prefixing with `pypi__`. Alternatively we could require + `--noexperimental_python_import_all_repositories` be set, however this breaks rules_docker. + See: https://github.com/bazelbuild/bazel/issues/2636 + """ + + return "pypi__" + name.replace("-", "_").replace(".", "_").lower() + + +def setup_namespace_pkg_compatibility(wheel_dir: str) -> None: + """Converts native namespace packages to pkgutil-style packages + + Namespace packages can be created in one of three ways. They are detailed here: + https://packaging.python.org/guides/packaging-namespace-packages/#creating-a-namespace-package + + 'pkgutil-style namespace packages' (2) and 'pkg_resources-style namespace packages' (3) works in Bazel, but + 'native namespace packages' (1) do not. + + We ensure compatibility with Bazel of method 1 by converting them into method 2. + + Args: + wheel_dir: the directory of the wheel to convert + """ + + namespace_pkg_dirs = namespace_pkgs.implicit_namespace_packages( + wheel_dir, ignored_dirnames=["%s/bin" % wheel_dir,], + ) + + for ns_pkg_dir in namespace_pkg_dirs: + namespace_pkgs.add_pkgutil_style_namespace_pkg_init(ns_pkg_dir) + + +def extract_wheel( + wheel_file: str, + extras: Dict[str, Set[str]], + pip_data_exclude: List[str], + enable_implicit_namespace_pkgs: bool, +) -> str: + """Extracts wheel into given directory and creates a py_library target. + + Args: + wheel_file: the filepath of the .whl + extras: a list of extras to add as dependencies for the installed wheel + pip_data_exclude: list of file patterns to exclude from the generated data section of the py_library + enable_implicit_namespace_pkgs: if true, disables conversion of implicit namespace packages and will unzip as-is + + Returns: + The Bazel label for the extracted wheel, in the form '//path/to/wheel'. + """ + + whl = wheel.Wheel(wheel_file) + directory = sanitise_name(whl.name) + + os.mkdir(directory) + whl.unzip(directory) + + # Note: Order of operations matters here + purelib.spread_purelib_into_root(directory) + + if not enable_implicit_namespace_pkgs: + setup_namespace_pkg_compatibility(directory) + + extras_requested = extras[whl.name] if whl.name in extras else set() + + sanitised_dependencies = [ + '"//%s"' % sanitise_name(d) for d in sorted(whl.dependencies(extras_requested)) + ] + + with open(os.path.join(directory, "BUILD"), "w") as build_file: + contents = generate_build_file_contents( + sanitise_name(whl.name), sanitised_dependencies, pip_data_exclude, + ) + build_file.write(contents) + + os.remove(whl.path) + + return "//%s" % directory diff --git a/experimental/rules_python_external/extract_wheels/lib/namespace_pkgs.py b/experimental/rules_python_external/extract_wheels/lib/namespace_pkgs.py new file mode 100644 index 0000000000..cb9e164b85 --- /dev/null +++ b/experimental/rules_python_external/extract_wheels/lib/namespace_pkgs.py @@ -0,0 +1,72 @@ +"""Utility functions to discover python package types""" +import os +import textwrap +from typing import Set, List, Optional + +from extract_wheels.lib import wheel + + +def implicit_namespace_packages( + directory: str, ignored_dirnames: Optional[List[str]] = None +) -> Set[str]: + """Discovers namespace packages implemented using the 'native namespace packages' method. + + AKA 'implicit namespace packages', which has been supported since Python 3.3. + See: https://packaging.python.org/guides/packaging-namespace-packages/#native-namespace-packages + + Args: + directory: The root directory to recursively find packages in. + ignored_dirnames: A list of directories to exclude from the search + + Returns: + The set of directories found under root to be packages using the native namespace method. + """ + namespace_pkg_dirs = set() + for dirpath, dirnames, filenames in os.walk(directory, topdown=True): + # We are only interested in dirs with no __init__.py file + if "__init__.py" in filenames: + dirnames[:] = [] # Remove dirnames from search + continue + + for ignored_dir in ignored_dirnames or []: + if ignored_dir in dirnames: + dirnames.remove(ignored_dir) + + non_empty_directory = dirnames or filenames + if ( + non_empty_directory + and + # The root of the directory should never be an implicit namespace + dirpath != directory + ): + namespace_pkg_dirs.add(dirpath) + + return namespace_pkg_dirs + + +def add_pkgutil_style_namespace_pkg_init(dir_path: str) -> None: + """Adds 'pkgutil-style namespace packages' init file to the given directory + + See: https://packaging.python.org/guides/packaging-namespace-packages/#pkgutil-style-namespace-packages + + Args: + dir_path: The directory to create an __init__.py for. + + Raises: + ValueError: If the directory already contains an __init__.py file + """ + ns_pkg_init_filepath = os.path.join(dir_path, "__init__.py") + + if os.path.isfile(ns_pkg_init_filepath): + raise ValueError("%s already contains an __init__.py file." % dir_path) + + with open(ns_pkg_init_filepath, "w") as ns_pkg_init_f: + # See https://packaging.python.org/guides/packaging-namespace-packages/#pkgutil-style-namespace-packages + ns_pkg_init_f.write( + textwrap.dedent( + """\ + # __path__ manipulation added by rules_python_external to support namespace pkgs. + __path__ = __import__('pkgutil').extend_path(__path__, __name__) + """ + ) + ) diff --git a/experimental/rules_python_external/extract_wheels/lib/namespace_pkgs_test.py b/experimental/rules_python_external/extract_wheels/lib/namespace_pkgs_test.py new file mode 100644 index 0000000000..899f7bc053 --- /dev/null +++ b/experimental/rules_python_external/extract_wheels/lib/namespace_pkgs_test.py @@ -0,0 +1,73 @@ +import pathlib +import shutil +import tempfile +from typing import Optional +import unittest + +from extract_wheels.lib import namespace_pkgs + + +class TempDir: + def __init__(self) -> None: + self.dir = tempfile.mkdtemp() + + def root(self) -> str: + return self.dir + + def add_dir(self, rel_path: str) -> None: + d = pathlib.Path(self.dir, rel_path) + d.mkdir(parents=True) + + def add_file(self, rel_path: str, contents: Optional[str] = None) -> None: + f = pathlib.Path(self.dir, rel_path) + f.parent.mkdir(parents=True, exist_ok=True) + if contents: + with open(str(f), "w") as writeable_f: + writeable_f.write(contents) + else: + f.touch() + + def remove(self) -> None: + shutil.rmtree(self.dir) + + +class TestImplicitNamespacePackages(unittest.TestCase): + def test_finds_correct_namespace_packages(self) -> None: + directory = TempDir() + directory.add_file("foo/bar/biz.py") + directory.add_file("foo/bee/boo.py") + directory.add_file("foo/buu/__init__.py") + directory.add_file("foo/buu/bii.py") + + expected = { + directory.root() + "/foo", + directory.root() + "/foo/bar", + directory.root() + "/foo/bee", + } + actual = namespace_pkgs.implicit_namespace_packages(directory.root()) + self.assertEqual(actual, expected) + + def test_ignores_empty_directories(self) -> None: + directory = TempDir() + directory.add_file("foo/bar/biz.py") + directory.add_dir("foo/cat") + + expected = { + directory.root() + "/foo", + directory.root() + "/foo/bar", + } + actual = namespace_pkgs.implicit_namespace_packages(directory.root()) + self.assertEqual(actual, expected) + + def test_empty_case(self) -> None: + directory = TempDir() + directory.add_file("foo/__init__.py") + directory.add_file("foo/bar/__init__.py") + directory.add_file("foo/bar/biz.py") + + actual = namespace_pkgs.implicit_namespace_packages(directory.root()) + self.assertEqual(actual, set()) + + +if __name__ == "__main__": + unittest.main() diff --git a/experimental/rules_python_external/extract_wheels/lib/purelib.py b/experimental/rules_python_external/extract_wheels/lib/purelib.py new file mode 100644 index 0000000000..ffcda8ffcc --- /dev/null +++ b/experimental/rules_python_external/extract_wheels/lib/purelib.py @@ -0,0 +1,56 @@ +"""Functions to make purelibs Bazel compatible""" +import pathlib +import shutil + +from extract_wheels.lib import wheel + + +def spread_purelib_into_root(wheel_dir: str) -> None: + """Unpacks purelib directories into the root. + + Args: + wheel_dir: The root of the extracted wheel directory. + """ + dist_info = wheel.get_dist_info(wheel_dir) + wheel_metadata_file_path = pathlib.Path(dist_info, "WHEEL") + wheel_metadata_dict = wheel.parse_wheel_meta_file(str(wheel_metadata_file_path)) + + if "Root-Is-Purelib" not in wheel_metadata_dict: + raise ValueError( + "Invalid WHEEL file '%s'. Expected key 'Root-Is-Purelib'." + % wheel_metadata_file_path + ) + root_is_purelib = wheel_metadata_dict["Root-Is-Purelib"] + + if root_is_purelib.lower() == "true": + # The Python package code is in the root of the Wheel, so no need to 'spread' anything. + return + + dot_data_dir = wheel.get_dot_data_directory(wheel_dir) + # 'Root-Is-Purelib: false' is no guarantee a .date directory exists with + # package code in it. eg. the 'markupsafe' package. + if not dot_data_dir: + return + + for child in pathlib.Path(dot_data_dir).iterdir(): + # TODO(Jonathon): Should all other potential folders get ignored? eg. 'platlib' + if str(child).endswith("purelib"): + _spread_purelib(child, wheel_dir) + + +def _spread_purelib(purelib_dir: pathlib.Path, root_dir: str) -> None: + """Recursively moves all sibling directories of the purelib to the root. + + Args: + purelib_dir: The directory of the purelib. + root_dir: The directory to move files into. + """ + for grandchild in purelib_dir.iterdir(): + # Some purelib Wheels, like Tensorflow 2.0.0, have directories + # split between the root and the purelib directory. In this case + # we should leave the purelib 'sibling' alone. + # See: https://github.com/dillon-giacoppo/rules_python_external/issues/8 + if not pathlib.Path(root_dir, grandchild.name).exists(): + shutil.move( + src=str(grandchild), dst=root_dir, + ) diff --git a/experimental/rules_python_external/extract_wheels/lib/requirements.py b/experimental/rules_python_external/extract_wheels/lib/requirements.py new file mode 100644 index 0000000000..e246379bcc --- /dev/null +++ b/experimental/rules_python_external/extract_wheels/lib/requirements.py @@ -0,0 +1,45 @@ +import re +from typing import Dict, Set, Tuple, Optional + + +def parse_extras(requirements_path: str) -> Dict[str, Set[str]]: + """Parse over the requirements.txt file to find extras requested. + + Args: + requirements_path: The filepath for the requirements.txt file to parse. + + Returns: + A dictionary mapping the requirement name to a set of extras requested. + """ + + extras_requested = {} + with open(requirements_path, "r") as requirements: + # Merge all backslash line continuations so we parse each requirement as a single line. + for line in requirements.read().replace("\\\n", "").split("\n"): + requirement, extras = _parse_requirement_for_extra(line) + if requirement and extras: + extras_requested[requirement] = extras + + return extras_requested + + +def _parse_requirement_for_extra( + requirement: str, +) -> Tuple[Optional[str], Optional[Set[str]]]: + """Given a requirement string, returns the requirement name and set of extras, if extras specified. + Else, returns (None, None) + """ + + # https://www.python.org/dev/peps/pep-0508/#grammar + extras_pattern = re.compile( + r"^\s*([0-9A-Za-z][0-9A-Za-z_.\-]*)\s*\[\s*([0-9A-Za-z][0-9A-Za-z_.\-]*(?:\s*,\s*[0-9A-Za-z][0-9A-Za-z_.\-]*)*)\s*\]" + ) + + matches = extras_pattern.match(requirement) + if matches: + return ( + matches.group(1), + {extra.strip() for extra in matches.group(2).split(",")}, + ) + + return None, None diff --git a/experimental/rules_python_external/extract_wheels/lib/requirements_test.py b/experimental/rules_python_external/extract_wheels/lib/requirements_test.py new file mode 100644 index 0000000000..2b96a754a0 --- /dev/null +++ b/experimental/rules_python_external/extract_wheels/lib/requirements_test.py @@ -0,0 +1,32 @@ +import unittest + +from extract_wheels.lib import requirements + + +class TestRequirementExtrasParsing(unittest.TestCase): + def test_parses_requirement_for_extra(self) -> None: + cases = [ + ("name[foo]", ("name", frozenset(["foo"]))), + ("name[ Foo123 ]", ("name", frozenset(["Foo123"]))), + (" name1[ foo ] ", ("name1", frozenset(["foo"]))), + ( + "name [fred,bar] @ http://foo.com ; python_version=='2.7'", + ("name", frozenset(["fred", "bar"])), + ), + ( + "name[quux, strange];python_version<'2.7' and platform_version=='2'", + ("name", frozenset(["quux", "strange"])), + ), + ("name; (os_name=='a' or os_name=='b') and os_name=='c'", (None, None),), + ("name@http://foo.com", (None, None),), + ] + + for case, expected in cases: + with self.subTest(): + self.assertTupleEqual( + requirements._parse_requirement_for_extra(case), expected + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/experimental/rules_python_external/extract_wheels/lib/wheel.py b/experimental/rules_python_external/extract_wheels/lib/wheel.py new file mode 100644 index 0000000000..c13f4e8621 --- /dev/null +++ b/experimental/rules_python_external/extract_wheels/lib/wheel.py @@ -0,0 +1,147 @@ +"""Utility class to inspect an extracted wheel directory""" +import glob +import os +import stat +import zipfile +from typing import Dict, Optional, Set + +import pkg_resources +import pkginfo + + +def current_umask() -> int: + """Get the current umask which involves having to set it temporarily.""" + mask = os.umask(0) + os.umask(mask) + return mask + + +def set_extracted_file_to_default_mode_plus_executable(path: str) -> None: + """ + Make file present at path have execute for user/group/world + (chmod +x) is no-op on windows per python docs + """ + os.chmod(path, (0o777 & ~current_umask() | 0o111)) + + +class Wheel: + """Representation of the compressed .whl file""" + + def __init__(self, path: str): + self._path = path + + @property + def path(self) -> str: + return self._path + + @property + def name(self) -> str: + return str(self.metadata.name) + + @property + def metadata(self) -> pkginfo.Wheel: + return pkginfo.get_metadata(self.path) + + def dependencies(self, extras_requested: Optional[Set[str]] = None) -> Set[str]: + dependency_set = set() + + for wheel_req in self.metadata.requires_dist: + req = pkg_resources.Requirement(wheel_req) # type: ignore + + if req.marker is None or any( + req.marker.evaluate({"extra": extra}) + for extra in extras_requested or [""] + ): + dependency_set.add(req.name) # type: ignore + + return dependency_set + + def unzip(self, directory: str) -> None: + with zipfile.ZipFile(self.path, "r") as whl: + whl.extractall(directory) + # The following logic is borrowed from Pip: + # https://github.com/pypa/pip/blob/cc48c07b64f338ac5e347d90f6cb4efc22ed0d0b/src/pip/_internal/utils/unpacking.py#L240 + for info in whl.infolist(): + name = info.filename + # Do not attempt to modify directories. + if name.endswith("/") or name.endswith("\\"): + continue + mode = info.external_attr >> 16 + # if mode and regular file and any execute permissions for + # user/group/world? + if mode and stat.S_ISREG(mode) and mode & 0o111: + name = os.path.join(directory, name) + set_extracted_file_to_default_mode_plus_executable(name) + + +def get_dist_info(wheel_dir: str) -> str: + """"Returns the relative path to the dist-info directory if it exists. + + Args: + wheel_dir: The root of the extracted wheel directory. + + Returns: + Relative path to the dist-info directory if it exists, else, None. + """ + dist_info_dirs = glob.glob(os.path.join(wheel_dir, "*.dist-info")) + if not dist_info_dirs: + raise ValueError( + "No *.dist-info directory found. %s is not a valid Wheel." % wheel_dir + ) + + if len(dist_info_dirs) > 1: + raise ValueError( + "Found more than 1 *.dist-info directory. %s is not a valid Wheel." + % wheel_dir + ) + + return dist_info_dirs[0] + + +def get_dot_data_directory(wheel_dir: str) -> Optional[str]: + """Returns the relative path to the data directory if it exists. + + See: https://www.python.org/dev/peps/pep-0491/#the-data-directory + + Args: + wheel_dir: The root of the extracted wheel directory. + + Returns: + Relative path to the data directory if it exists, else, None. + """ + + dot_data_dirs = glob.glob(os.path.join(wheel_dir, "*.data")) + if not dot_data_dirs: + return None + + if len(dot_data_dirs) > 1: + raise ValueError( + "Found more than 1 *.data directory. %s is not a valid Wheel." % wheel_dir + ) + + return dot_data_dirs[0] + + +def parse_wheel_meta_file(wheel_dir: str) -> Dict[str, str]: + """Parses the given WHEEL file into a dictionary. + + Args: + wheel_dir: The file path of the WHEEL metadata file in dist-info. + + Returns: + The WHEEL file mapped into a dictionary. + """ + contents = {} + with open(wheel_dir, "r") as wheel_file: + for line in wheel_file: + cleaned = line.strip() + if not cleaned: + continue + try: + key, value = cleaned.split(":", maxsplit=1) + contents[key] = value.strip() + except ValueError: + raise RuntimeError( + "Encounted invalid line in WHEEL file: '%s'" % cleaned + ) + return contents diff --git a/experimental/rules_python_external/repositories.bzl b/experimental/rules_python_external/repositories.bzl new file mode 100644 index 0000000000..8fb146607d --- /dev/null +++ b/experimental/rules_python_external/repositories.bzl @@ -0,0 +1,57 @@ +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") + +_RULE_DEPS = [ + ( + "pypi__pip", + "https://files.pythonhosted.org/packages/00/b6/9cfa56b4081ad13874b0c6f96af8ce16cfbc1cb06bedf8e9164ce5551ec1/pip-19.3.1-py2.py3-none-any.whl", + "6917c65fc3769ecdc61405d3dfd97afdedd75808d200b2838d7d961cebc0c2c7", + ), + ( + "pypi__pkginfo", + "https://files.pythonhosted.org/packages/e6/d5/451b913307b478c49eb29084916639dc53a88489b993530fed0a66bab8b9/pkginfo-1.5.0.1-py2.py3-none-any.whl", + "a6d9e40ca61ad3ebd0b72fbadd4fba16e4c0e4df0428c041e01e06eb6ee71f32", + ), + ( + "pypi__setuptools", + "https://files.pythonhosted.org/packages/54/28/c45d8b54c1339f9644b87663945e54a8503cfef59cf0f65b3ff5dd17cf64/setuptools-42.0.2-py2.py3-none-any.whl", + "c8abd0f3574bc23afd2f6fd2c415ba7d9e097c8a99b845473b0d957ba1e2dac6", + ), + ( + "pypi__wheel", + "https://files.pythonhosted.org/packages/00/83/b4a77d044e78ad1a45610eb88f745be2fd2c6d658f9798a15e384b7d57c9/wheel-0.33.6-py2.py3-none-any.whl", + "f4da1763d3becf2e2cd92a14a7c920f0f00eca30fdde9ea992c836685b9faf28", + ), +] + +_GENERIC_WHEEL = """\ +package(default_visibility = ["//visibility:public"]) + +load("@rules_python//python:defs.bzl", "py_library") + +py_library( + name = "lib", + srcs = glob(["**/*.py"]), + data = glob(["**/*"], exclude=["**/*.py", "**/* *", "BUILD", "WORKSPACE"]), + # This makes this directory a top-level in the python import + # search path for anything that depends on this. + imports = ["."], +) +""" + +# Collate all the repository names so they can be easily consumed +all_requirements = [name for (name, _, _) in _RULE_DEPS] + +def requirement(pkg): + return "@pypi__"+ pkg + "//:lib" + +def rules_python_external_dependencies(): + for (name, url, sha256) in _RULE_DEPS: + maybe( + http_archive, + name, + url=url, + sha256=sha256, + type="zip", + build_file_content=_GENERIC_WHEEL, + ) diff --git a/experimental/rules_python_external/tools/typing/BUILD b/experimental/rules_python_external/tools/typing/BUILD new file mode 100644 index 0000000000..e69de29bb2 diff --git a/experimental/rules_python_external/tools/typing/mypy.ini b/experimental/rules_python_external/tools/typing/mypy.ini new file mode 100644 index 0000000000..ed392f2eeb --- /dev/null +++ b/experimental/rules_python_external/tools/typing/mypy.ini @@ -0,0 +1,24 @@ +[mypy] +# This should be the oldest supported release of Python +# https://devguide.python.org/#status-of-python-branches +python_version = 3.5 + +# Third-Party packages without Stub files +# https://mypy.readthedocs.io/en/latest/stubs.html +[mypy-pkginfo.*] +ignore_missing_imports = True + +[mypy-extract_wheels.*] +check_untyped_defs = True +disallow_incomplete_defs = True +disallow_untyped_calls = True +disallow_untyped_decorators = True +disallow_untyped_defs = True +no_implicit_optional = True +strict_equality = True +strict_optional = True +warn_no_return = True +warn_redundant_casts = True +warn_return_any = True +warn_unreachable = True +warn_unused_ignores = True diff --git a/experimental/rules_python_external/tools/typing/mypy_version.txt b/experimental/rules_python_external/tools/typing/mypy_version.txt new file mode 100644 index 0000000000..9e7ef885f9 --- /dev/null +++ b/experimental/rules_python_external/tools/typing/mypy_version.txt @@ -0,0 +1 @@ +mypy==0.780