diff --git a/README.md b/README.md index fba35a05..8e443d56 100644 --- a/README.md +++ b/README.md @@ -341,6 +341,31 @@ Sometimes you want to use Conan as [in-source](https://docs.conan.io/en/latest/c As SHA-1 is 40 digits long, you could format the result to short size +## Save created packages summary +In case you want to integrate CPT with other tools, for example you want to have build logic after creating packages, you can save a json report about all configurations and packages. + +**Examples**: + + from cpt.packager import ConanMultiPackager + + if __name__ == "__main__": + builder = ConanMultiPackager() + builder.add_common_builds() + builder.run(summary_file='cpt_summary_file.json') + + + from cpt.packager import ConanMultiPackager + + if __name__ == "__main__": + builder = ConanMultiPackager() + builder.add_common_builds() + builder.run() + builder.save_packages_summary('cpt_summary_file.json') + + +Alternatively you can use the `CPT_SUMMARY_FILE` environment variable to set the summary file path + + ## Using Docker @@ -1072,6 +1097,7 @@ Using **CONAN_CLANG_VERSIONS** env variable in Travis ci or Appveyor: - "missing": Build only missing packages. - "outdated": Build only missing or if the available package is not built with the current recipe. Useful to upload new configurations, e.j packages for a new compiler without rebuild all packages. + - "all": Build all requirements. - **test_folder**: Custom test folder consumed by Conan create, e.j .conan/test_package - **conanfile**: Custom conanfile consumed by Conan create. e.j. conanfile.py - **config_url**: Conan config URL be installed before to build e.j https://github.com/bincrafters/conan-config.git @@ -1178,6 +1204,7 @@ This is especially useful for CI integration. - **CONAN_APPLE_CLANG_VERSIONS**: Apple clang versions, comma separated, e.g. "6.1,8.0" - **CONAN_ARCHS**: Architectures to build for, comma separated, e.g. "x86,x86_64" - **CONAN_OPTIONS**: Conan build options, comma separated, e.g. "foobar:with_bar=True,foobar:with_qux=False" +- **CONAN_SHARED_OPTION_NAME**: Set `shared_option_name` by environment variable, e.g. "mypackagename:shared" - **CONAN_BUILD_TYPES**: Build types to build for, comma separated, e.g. "Release,Debug" - **CONAN_CPPSTDS**: List containing values for `compiler.cppstd`. Default None - **CONAN_VISUAL_VERSIONS**: Visual versions, comma separated, e.g. "12,14" diff --git a/appveyor.yml b/appveyor.yml index 187325e8..0ec57147 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -2,8 +2,10 @@ environment: matrix: - PYTHON: "C:\\Python27" APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 + USE_UNSUPPORTED_CONAN_WITH_PYTHON_2: "1" - PYTHON: "C:\\Python35" APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 + USE_UNSUPPORTED_CONAN_WITH_PYTHON_2: "1" build: false install: diff --git a/cpt/__init__.py b/cpt/__init__.py index 066819da..cca79aad 100644 --- a/cpt/__init__.py +++ b/cpt/__init__.py @@ -1,6 +1,6 @@ -__version__ = '0.30.4' -NEWEST_CONAN_SUPPORTED = "1.21.100" +__version__ = '0.31.0' +NEWEST_CONAN_SUPPORTED = "1.22.000" def get_client_version(): diff --git a/cpt/ci_manager.py b/cpt/ci_manager.py index d4855aff..aa4e6f5e 100644 --- a/cpt/ci_manager.py +++ b/cpt/ci_manager.py @@ -68,7 +68,7 @@ def get_commit_build_policy(self): matches = prog.match(msg) if matches: build_policy = matches.groups()[0] - if build_policy not in ("never", "outdated", "missing"): + if build_policy not in ("never", "outdated", "missing", "all"): raise Exception("Invalid build policy, valid values: never, outdated, missing") return build_policy return None diff --git a/cpt/packager.py b/cpt/packager.py index 36692fde..e45ae7ad 100644 --- a/cpt/packager.py +++ b/cpt/packager.py @@ -26,21 +26,22 @@ def load_cf_class(path, conan_api): client_version = get_client_version() - if Version(client_version) < Version("1.7.0"): + client_version = Version(client_version) + if client_version < Version("1.7.0"): from conans.client.loader_parse import load_conanfile_class return load_conanfile_class(path) - elif Version(client_version) < Version("1.14.0"): + elif client_version < Version("1.14.0"): return conan_api._loader.load_class(path) - elif Version(client_version) < Version("1.15.0"): + elif client_version < Version("1.15.0"): remotes = conan_api._cache.registry.remotes.list for remote in remotes: conan_api.python_requires.enable_remotes(remote_name=remote) return conan_api._loader.load_class(path) - elif Version(client_version) < Version("1.16.0"): + elif client_version < Version("1.16.0"): remotes = conan_api._cache.registry.load_remotes() conan_api.python_requires.enable_remotes(remotes=remotes) return conan_api._loader.load_class(path) - elif Version(client_version) < Version("1.18.0"): + elif client_version < Version("1.18.0"): remotes = conan_api._cache.registry.load_remotes() conan_api._python_requires.enable_remotes(remotes=remotes) return conan_api._loader.load_class(path) @@ -49,10 +50,13 @@ def load_cf_class(path, conan_api): conan_api.create_app() remotes = conan_api.app.cache.registry.load_remotes() conan_api.app.python_requires.enable_remotes(remotes=remotes) - if Version(client_version) < Version("1.20.0"): + conan_api.app.pyreq_loader.enable_remotes(remotes=remotes) + if client_version < Version("1.20.0"): return conan_api.app.loader.load_class(path) - else: + elif client_version < Version("1.21.0"): return conan_api.app.loader.load_basic(path) + else: + return conan_api.app.loader.load_named(path, None, None, None, None) class PlatformInfo(object): @@ -184,6 +188,7 @@ def __init__(self, username=None, channel=None, runner=None, self._builds = [] self._named_builds = {} + self._packages_summary = [] self._update_conan_in_docker = always_update_conan_in_docker or get_bool_from_env("CONAN_ALWAYS_UPDATE_CONAN_DOCKER") @@ -237,7 +242,7 @@ def __init__(self, username=None, channel=None, runner=None, os.getenv("CONAN_BUILD_POLICY", None)) if build_policy: - if build_policy.lower() not in ("never", "outdated", "missing"): + if build_policy.lower() not in ("never", "outdated", "missing", "all"): raise Exception("Invalid build policy, valid values: never, outdated, missing") self.build_policy = build_policy @@ -371,6 +376,21 @@ def container_os(self): else: return "" + @property + def packages_summary(self): + return self._packages_summary + + def save_packages_summary(self, file): + self.printer.print_message("Saving packages summary to " + file) + import json + import datetime + def default(o): + if isinstance(o, (datetime.date, datetime.datetime)): + return o.isoformat() + + with open(file, 'w') as outfile: + json.dump(self.packages_summary, outfile, default = default) + @property def items(self): return self._builds @@ -445,6 +465,10 @@ def add_common_builds(self, shared_option_name=None, pure_c=True, if not reference: raise Exception("Specify a CONAN_REFERENCE or name and version fields in the recipe") + if shared_option_name is None: + env_shared_option_name = os.getenv("CONAN_SHARED_OPTION_NAME", None) + shared_option_name = env_shared_option_name if str(env_shared_option_name).lower() != "false" else False + if shared_option_name is None: if os.path.exists(os.path.join(self.cwd, self.conanfile)): conanfile = load_cf_class(os.path.join(self.cwd, self.conanfile), self.conan_api) @@ -491,7 +515,7 @@ def update_build_if(self, predicate, new_settings=None, new_options=None, new_en updated_builds.append(build) self._builds = updated_builds - def run(self, base_profile_name=None): + def run(self, base_profile_name=None, summary_file=None): self._check_conan_version() env_vars = self.auth_manager.env_vars() @@ -516,6 +540,10 @@ def run(self, base_profile_name=None): self.run_builds(base_profile_name=base_profile_name) + summary_file = summary_file or os.getenv("CPT_SUMMARY_FILE", None) + if summary_file: + self.save_packages_summary(summary_file) + def _upload_enabled(self): if not self.remotes_manager.upload_remote_name: return False @@ -603,6 +631,7 @@ def run_builds(self, curpage=None, total_pages=None, base_profile_name=None): skip_recipe_export=skip_recipe_export, update_dependencies=self.update_dependencies) r.run() + self._packages_summary.append({"configuration": build, "package" : r.results}) else: docker_image = self._get_docker_image(build) r = DockerCreateRunner(profile_text, base_profile_text, base_profile_name, diff --git a/cpt/requirements.txt b/cpt/requirements.txt index 4ad4d320..e3d2b9cc 100644 --- a/cpt/requirements.txt +++ b/cpt/requirements.txt @@ -1,3 +1,3 @@ six>=1.10.0, <1.13.0 -conan>=1.7.0, <1.22.0 +conan>=1.7.0, <1.23.0 tabulate==0.8.2 diff --git a/cpt/runner.py b/cpt/runner.py index d2629e85..ee77fb4f 100644 --- a/cpt/runner.py +++ b/cpt/runner.py @@ -43,6 +43,7 @@ def __init__(self, profile_abs_path, reference, conan_api, uploader, self._upload_dependencies = self._upload_dependencies or [] self.skip_recipe_export = skip_recipe_export self._update_dependencies = update_dependencies + self._results = None patch_default_base_profile(conan_api, profile_abs_path) client_version = get_client_version() @@ -62,6 +63,10 @@ def __init__(self, profile_abs_path, reference, conan_api, uploader, def settings(self): return self._profile.settings + @property + def results(self): + return self._results + def run(self): client_version = get_client_version() @@ -88,7 +93,7 @@ def run(self): name, version, user, channel, _ = self._reference if self._build_policy: - self._build_policy = [self._build_policy] + self._build_policy = [] if self._build_policy == "all" else [self._build_policy] # https://github.com/conan-io/conan-package-tools/issues/184 with tools.environment_append({"_CONAN_CREATE_COMMAND_": "1"}): params = {"name": name, "version": version, "user": user, @@ -105,7 +110,7 @@ def run(self): try: if client_version < Version("1.12.0"): - r = self._conan_api.create(self._conanfile, name=name, version=version, + self._results = self._conan_api.create(self._conanfile, name=name, version=version, user=user, channel=channel, build_modes=self._build_policy, profile_name=self._profile_abs_path, @@ -113,7 +118,7 @@ def run(self): not_export=self.skip_recipe_export, update=self._update_dependencies) else: - r = self._conan_api.create(self._conanfile, name=name, version=version, + self._results = self._conan_api.create(self._conanfile, name=name, version=version, user=user, channel=channel, build_modes=self._build_policy, profile_names=[self._profile_abs_path], @@ -126,7 +131,7 @@ def run(self): "%s" % str(e)) self.printer.print_rule() return - for installed in r['installed']: + for installed in self._results['installed']: reference = installed["recipe"]["id"] if client_version >= Version("1.10.0"): reference = ConanFileReference.loads(reference) diff --git a/cpt/test/integration/basic_test.py b/cpt/test/integration/basic_test.py index 8f94f076..babc1ccb 100644 --- a/cpt/test/integration/basic_test.py +++ b/cpt/test/integration/basic_test.py @@ -1,6 +1,7 @@ import os import unittest import sys +import json from conans import tools from conans.model.ref import ConanFileReference @@ -223,3 +224,62 @@ def configure(self): self.packager.add({}, {}, {}, {}) self.packager.run() self.assertIn("partial_reference | foobar/0.1.0@", self.output) + + def test_save_packages_summary(self): + conanfile = """from conans import ConanFile +class Pkg(ConanFile): + name = "foobar" + version = "0.1.0" + + def configure(self): + self.output.info("hello all") +""" + json_file = 'cpt_summary_file.json' + tools.save(os.path.join(self.tmp_folder, "conanfile.py"), conanfile) + self.packager = ConanMultiPackager(out=self.output.write) + self.packager.add({}, {}, {}, {}) + self.packager.run(summary_file=json_file) + self.assertTrue(os.path.isfile(json_file)) + with open(json_file) as json_content: + json_data = json.load(json_content) + self.assertFalse(json_data[0]["package"]["error"]) + + json_file = "_" + json_file + self.packager = ConanMultiPackager(out=self.output.write) + self.packager.add({}, {}, {}, {}) + self.packager.run() + self.packager.save_packages_summary(json_file) + self.assertTrue(os.path.isfile(json_file)) + with open(json_file) as json_content: + json_data = json.load(json_content) + self.assertFalse(json_data[0]["package"]["error"]) + + json_file = "__" + json_file + with tools.environment_append({"CPT_SUMMARY_FILE": json_file}): + self.packager = ConanMultiPackager(out=self.output.write) + self.packager.add({}, {}, {}, {}) + self.packager.run() + self.assertTrue(os.path.isfile(json_file)) + with open(json_file) as json_content: + json_data = json.load(json_content) + self.assertFalse(json_data[0]["package"]["error"]) + + def test_custom_name_version(self): + conanfile = """from conans import ConanFile +from datetime import date +class Pkg(ConanFile): + + def configure(self): + self.output.info("hello all") + + def set_name(self): + self.name = "foobar" + + def set_version(self): + today = date.today() + self.version = today.strftime("%Y%B%d") +""" + tools.save(os.path.join(self.tmp_folder, "conanfile.py"), conanfile) + self.packager = ConanMultiPackager(out=self.output.write) + self.packager.add_common_builds(pure_c=False) + self.packager.run() diff --git a/cpt/test/integration/update_python_reqs_test.py b/cpt/test/integration/update_python_reqs_test.py new file mode 100644 index 00000000..5f0a177d --- /dev/null +++ b/cpt/test/integration/update_python_reqs_test.py @@ -0,0 +1,44 @@ +import unittest + +from conans.test.utils.tools import TestClient +from cpt.test.test_client.tools import get_patched_multipackager + + +class PythonRequiresTest(unittest.TestCase): + + def test_python_requires(self): + base_conanfile = """from conans import ConanFile +myvar = 123 + +def myfunct(): + return 234 + +class Pkg(ConanFile): + pass +""" + + conanfile = """from conans import ConanFile + +class Pkg(ConanFile): + name = "pyreq" + version = "1.0.0" + python_requires = "pyreq_base/0.1@user/channel" + + def build(self): + v = self.python_requires["pyreq_base"].module.myvar + f = self.python_requires["pyreq_base"].module.myfunct() + self.output.info("%s,%s" % (v, f)) +""" + + client = TestClient() + client.save({"conanfile_base.py": base_conanfile}) + client.run("export conanfile_base.py pyreq_base/0.1@user/channel") + + client.save({"conanfile.py": conanfile}) + mulitpackager = get_patched_multipackager(client, username="user", + channel="testing", + exclude_vcvars_precommand=True) + mulitpackager.add({}, {}) + mulitpackager.run() + self.assertIn("pyreq/1.0.0@user/", client.out) + self.assertIn(": 123,234", client.out) diff --git a/cpt/test/unit/ci_manager_test.py b/cpt/test/unit/ci_manager_test.py index 63b6e308..d95d3f0e 100644 --- a/cpt/test/unit/ci_manager_test.py +++ b/cpt/test/unit/ci_manager_test.py @@ -206,6 +206,13 @@ def test_build_policy(self): self.assertEquals(manager.get_commit_msg(), "This is a great commit " "[build=outdated] End.") + with tools.environment_append({"TRAVIS": "1", + "TRAVIS_COMMIT_MESSAGE": + "This is a great commit [build=all] End."}): + manager = CIManager(self.printer) + self.assertEquals(manager.get_commit_build_policy(), "all") + self.assertEquals(manager.get_commit_msg(), "This is a great commit " + "[build=all] End.") # Appveyor with tools.environment_append({"APPVEYOR": "1", "APPVEYOR_PULL_REQUEST_NUMBER": "1", diff --git a/cpt/test/unit/config_test.py b/cpt/test/unit/config_test.py index 0c2bb52b..035b07c5 100644 --- a/cpt/test/unit/config_test.py +++ b/cpt/test/unit/config_test.py @@ -15,11 +15,11 @@ def setUp(self): def test_valid_config(self): manager = ConfigManager(self.conan_api, Printer()) - manager.install('https://github.com/bincrafters/conan-config.git') + manager.install('https://github.com/bincrafters/bincrafters-config.git') def test_valid_config_with_args(self): manager = ConfigManager(self.conan_api, Printer()) - manager.install('https://github.com/bincrafters/conan-config.git', '-b master') + manager.install('https://github.com/bincrafters/bincrafters-config.git', '-b master') class RemotesTestRealApi(BaseTest): @@ -30,7 +30,7 @@ def test_valid_config(self): profiles = self.api.profile_list() self.assertEquals(len(profiles), 0) - manager.install("https://github.com/bincrafters/conan-config.git", "-b master") + manager.install("https://github.com/bincrafters/bincrafters-config.git", "-b master") profiles = self.api.profile_list() self.assertGreater(len(profiles), 3) diff --git a/cpt/test/unit/packager_test.py b/cpt/test/unit/packager_test.py index cc5daabe..02dd1578 100644 --- a/cpt/test/unit/packager_test.py +++ b/cpt/test/unit/packager_test.py @@ -412,7 +412,8 @@ def test_only_mingw(self): builder = ConanMultiPackager(mingw_configurations=mingw_configurations, visual_versions=[], username="Pepe", platform_info=platform_mock_for("Windows"), reference="lib/1.0", ci_manager=self.ci_manager) - builder.add_common_builds(shared_option_name="zlib:shared", pure_c=True) + with tools.environment_append({"CONAN_SHARED_OPTION_NAME": "zlib:shared"}): + builder.add_common_builds(pure_c=True) expected = [({'compiler.exception': 'seh', 'compiler.libcxx': "libstdc++", 'compiler.threads': 'posix', 'compiler.version': '4.9', 'arch': 'x86_64', 'build_type': 'Release', 'compiler': 'gcc'}, @@ -445,10 +446,11 @@ def test_named_pages(self): builder = ConanMultiPackager(username="Pepe", reference="zlib/1.2.11", ci_manager=self.ci_manager) named_builds = defaultdict(list) - builder.add_common_builds(shared_option_name="zlib:shared", pure_c=True) - for settings, options, env_vars, build_requires, _ in builder.items: - named_builds[settings['arch']].append([settings, options, env_vars, build_requires]) - builder.named_builds = named_builds + with tools.environment_append({"CONAN_SHARED_OPTION_NAME": "zlib:shared"}): + builder.add_common_builds(pure_c=True) + for settings, options, env_vars, build_requires, _ in builder.items: + named_builds[settings['arch']].append([settings, options, env_vars, build_requires]) + builder.named_builds = named_builds self.assertEquals(builder.builds, []) if platform.system() == "Darwin": # Not default x86 in Macos @@ -652,21 +654,22 @@ def test_build_policy(self): builder.run() self.assertEquals(["outdated"], self.conan_api.calls[-1].kwargs["build_modes"]) - with tools.environment_append({"CONAN_BUILD_POLICY": "missing"}): - self.conan_api = MockConanAPI() - builder = ConanMultiPackager(username="pepe", channel="testing", - reference="Hello/0.1", password="password", - visual_versions=[], gcc_versions=[], - apple_clang_versions=[], - runner=self.runner, - conan_api=self.conan_api, - remotes="otherurl", - platform_info=platform_mock_for("Darwin"), - build_policy="missing", - ci_manager=self.ci_manager) - builder.add_common_builds() - builder.run() - self.assertEquals(["missing"], self.conan_api.calls[-1].kwargs["build_modes"]) + for build_policy, expected in [("missing", ["missing"]), ("all",[])]: + with tools.environment_append({"CONAN_BUILD_POLICY": build_policy}): + self.conan_api = MockConanAPI() + builder = ConanMultiPackager(username="pepe", channel="testing", + reference="Hello/0.1", password="password", + visual_versions=[], gcc_versions=[], + apple_clang_versions=[], + runner=self.runner, + conan_api=self.conan_api, + remotes="otherurl", + platform_info=platform_mock_for("Darwin"), + build_policy=build_policy, + ci_manager=self.ci_manager) + builder.add_common_builds() + builder.run() + self.assertEquals(expected, self.conan_api.calls[-1].kwargs["build_modes"]) def test_test_folder(self): builder = ConanMultiPackager(username="pepe", channel="testing", diff --git a/tox.ini b/tox.ini index 95cba190..f771a6e1 100644 --- a/tox.ini +++ b/tox.ini @@ -16,6 +16,7 @@ setenv = PYTHONPATH = {toxinidir}:{env:PYTHONPATH:} CONAN_TEST_SUITE=1 CONAN_DOCKER_USE_SUDO=1 + USE_UNSUPPORTED_CONAN_WITH_PYTHON_2=1 commands = nosetests --with-coverage --cover-package=cpt --cover-html {posargs:cpt.test} \ No newline at end of file