From aa307f58a97596f7ed9639e3c5c90021b29d8c32 Mon Sep 17 00:00:00 2001 From: Richard Jackson Date: Tue, 30 Jan 2018 22:30:05 -0600 Subject: [PATCH 1/7] Unpin setuptools Trying to fix downstream dependencies. Setuptools v30.X is quite old at this point, let's see if unpinning helps anything. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 48b30ad..116f9ce 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ license="Apache", packages=["cdisutils"], install_requires=[ - 'setuptools==30.1.0', + 'setuptools', 'xmltodict==0.9.2', 'pyOpenSSL==16.2.0', 'openpyxl==2.4.0', From f54e393c89939b2200dfae45c6235cbe2bae1206 Mon Sep 17 00:00:00 2001 From: Richard Jackson Date: Tue, 30 Jan 2018 22:47:28 -0600 Subject: [PATCH 2/7] Update signpost pin The newest signpost has an updated psycopg2, which fixes a build error. --- dev-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index c9693dc..6bd23b2 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -3,4 +3,4 @@ moto==0.4.1 nose==1.3.7 mock==1.3.0 python-novaclient==3.2.0 --e git+https://github.com/NCI-GDC/signpost.git@9d48b163d47cf059b9ba38a1b4562b2809f4ad2e#egg=signpost +-e git+https://github.com/NCI-GDC/signpost.git@v1.1#egg=signpost From 84f91c920ffa0205d0bd94f238aab4b2c0fb90ea Mon Sep 17 00:00:00 2001 From: Joe Sislow Date: Sat, 13 Jan 2018 01:07:44 +0000 Subject: [PATCH 3/7] feat(pins): Add functionality to update pins tool - Add arg for showing git in browser (made default false) - Add arg for default branch, have it override table - Add arg for dry_run so tmp dir is saved and push isn't done --- bin/update_dictionary_dependency_chain.py | 102 ++++++++++++++++------ 1 file changed, 74 insertions(+), 28 deletions(-) diff --git a/bin/update_dictionary_dependency_chain.py b/bin/update_dictionary_dependency_chain.py index c15ebbb..a5e4d3d 100644 --- a/bin/update_dictionary_dependency_chain.py +++ b/bin/update_dictionary_dependency_chain.py @@ -88,10 +88,21 @@ 'legacy-import': 'git@github.com:NCI-GDC/legacy-import.git', } +# edit this if you want to change each branch manually, otherwise +# just use the command line arg default_base_branch BASE_BRANCH_MAP = { - 'authoriation': 'origin/master', + 'gdcdatamodel': 'a31edc57056eaf7d71433773d4f030ad0303c1e3', + 'gdcapi': '4ce0f783a736cc8021cb676aaf809e0cfff621cb', + 'zugs': 'origin/develop', + 'esbuild': 'origin/develop', + 'runners': 'origin/develop', + 'auto-qa': 'origin/develop', + 'authorization': 'origin/develop', + 'legacy-import': 'origin/master', } +DEFAULT_BASE_BRANCH = 'origin/develop' + @contextmanager def within_dir(path): original_path = os.getcwd() @@ -105,7 +116,7 @@ def within_dir(path): @contextmanager -def within_tempdir(): +def within_tempdir(dry_run=False): original_path = os.getcwd() try: dirpath = tempfile.mkdtemp() @@ -113,8 +124,11 @@ def within_tempdir(): os.chdir(dirpath) yield dirpath finally: - print "Cleaning up temp files in %s" % dirpath - shutil.rmtree(dirpath) + if not dry_run: + print "Cleaning up temp files in %s" % dirpath + shutil.rmtree(dirpath) + else: + print "dry_run, saving temp files in {}".format(dirpath) os.chdir(original_path) @@ -132,16 +146,20 @@ def replace_dep_in_file(path, pattern, repl): with open(path, 'w') as updated: updated.write(data) -def get_base_branch(repo): +def get_b_idase_branch(repo): if repo in BASE_BRANCH_MAP: return BASE_BRANCH_MAP[repo] else: - return 'origin/develop' + return DEFAULT_BASE_BRANCH -def checkout_fresh_branch(repo, name): +def checkout_fresh_branch(repo, name, default_base=DEFAULT_BASE_BRANCH): cwd = os.getcwd() try: - base_branch = get_base_branch(repo) + #base_branch = get_base_branch(repo) + if default_base != DEFAULT_BASE_BRANCH: + base_branch = default_base + else: + base_branch = BASE_BRANCH_MAP.get(repo, default_base) print "Checking out new branch %s based off %s in %s" % (name, base_branch, repo) os.chdir(repo) @@ -151,37 +169,50 @@ def checkout_fresh_branch(repo, name): finally: os.chdir(cwd) -def commit_and_push(hash, branch): +def commit_and_push(hash, branch, dry_run=False): message = 'updating dictionary commit to %s' % hash check_call(['git', 'commit', '-am', message]) - print "Pushing datamodel origin/%s" % branch - check_call(['git', 'push', 'origin', branch]) + if not dry_run: + print "Pushing datamodel origin/%s" % branch + check_call(['git', 'push', 'origin', branch]) + else: + print "dry_run requested, skipping push" -def open_repo_url(): - proc = Popen(['git', 'config', '--get', 'remote.origin.url'] ,stdout=PIPE) - url = proc.stdout.read().replace('git@github.com:', 'https://github.com/') - print "Opening remote url %s" % url - call([OPEN_CMD, url]) +def open_repo_url(open_repo=False): + if open_repo: + proc = Popen(['git', 'config', '--get', 'remote.origin.url'] ,stdout=PIPE) + url = proc.stdout.read().replace('git@github.com:', 'https://github.com/') + print "Opening remote url %s" % url + call([OPEN_CMD, url]) -def bump_datamodel(branch, to_dictionary_hash): +def bump_datamodel(branch, + to_dictionary_hash, + open_repo=False, + dry_run=False, + default_base=DEFAULT_BASE_BRANCH): pattern = DEP_PIN_PATTERN.format(repo='gdcdictionary') repo = 'gdcdatamodel' url = REPO_MAP[repo] check_call(['git', 'clone', url]) - checkout_fresh_branch(repo, branch) + checkout_fresh_branch(repo, branch, default_base=default_base) with within_dir(repo): for path in DEPENDENCY_MAP[repo]: replace_dep_in_file(path, pattern, to_dictionary_hash) - commit_and_push(hash=to_dictionary_hash, branch=branch) - open_repo_url() + commit_and_push(hash=to_dictionary_hash, branch=branch, dry_run=dry_run) + open_repo_url(open_repo=open_repo) -def bump_downstream(branch, to_dictionary_hash, to_datamodel_hash): +def bump_downstream(branch, + to_dictionary_hash, + to_datamodel_hash, + open_repo=False, + dry_run=False, + default_base=DEFAULT_BASE_BRANCH): dictionary_pattern = DEP_PIN_PATTERN.format(repo='gdcdictionary') datamodel_pattern = DEP_PIN_PATTERN.format(repo='gdcdatamodel') @@ -190,7 +221,7 @@ def bump_downstream(branch, to_dictionary_hash, to_datamodel_hash): continue # should be done via bump_datamodel check_call(['git', 'clone', url]) - checkout_fresh_branch(repo, branch) + checkout_fresh_branch(repo, branch, default_base=default_base) with within_dir(repo): @@ -205,8 +236,8 @@ def bump_downstream(branch, to_dictionary_hash, to_datamodel_hash): datamodel_pattern, to_datamodel_hash) - commit_and_push(hash=to_dictionary_hash, branch=branch) - open_repo_url() + commit_and_push(hash=to_dictionary_hash, branch=branch, dry_run=dry_run) + open_repo_url(open_repo=open_repo) def main(): parser = argparse.ArgumentParser( @@ -218,20 +249,35 @@ def main(): parser.add_argument('--branch', help='branch to push bump as') parser.add_argument('--dictionary_commit', required=True, help='commit of dictionary') parser.add_argument('--datamodel_commit', required=False, help='commit of datamodel') + parser.add_argument('--open_browser', action='store_true', + help='open repo in browser after commit') + parser.add_argument('--default_base_branch', + help='default base branch to use for target repo, default = {}'.format(DEFAULT_BASE_BRANCH), + default=DEFAULT_BASE_BRANCH) + parser.add_argument('--dry_run', action='store_true', + help='dry run, simply create branches, do not push them') args = parser.parse_args() - with within_tempdir(): + with within_tempdir(dry_run=args.dry_run): if args.target == 'datamodel': - bump_datamodel(args.branch, args.dictionary_commit) + bump_datamodel(args.branch, + args.dictionary_commit, + open_repo=args.open_browser, + dry_run=args.dry_run, + default_base=args.default_base_branch) else: assert args.datamodel_commit, ( "When run with target=%s, argument `datamodel_commit` " "is required") % args.target - bump_downstream( - args.branch, args.dictionary_commit, args.datamodel_commit) + bump_downstream(args.branch, + args.dictionary_commit, + args.datamodel_commit, + open_repo=args.open_browser, + dry_run=args.dry_run, + default_base=args.default_base_branch) if __name__ == '__main__': main() From ce0bdd41eda2bc9a7f0a91496f05f1620c0a6be4 Mon Sep 17 00:00:00 2001 From: Joe Sislow Date: Fri, 12 Jan 2018 19:11:01 -0600 Subject: [PATCH 4/7] Update update_dictionary_dependency_chain.py Fix typo in def for get_base_branch --- bin/update_dictionary_dependency_chain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/update_dictionary_dependency_chain.py b/bin/update_dictionary_dependency_chain.py index a5e4d3d..5c718d0 100644 --- a/bin/update_dictionary_dependency_chain.py +++ b/bin/update_dictionary_dependency_chain.py @@ -146,7 +146,7 @@ def replace_dep_in_file(path, pattern, repl): with open(path, 'w') as updated: updated.write(data) -def get_b_idase_branch(repo): +def get_base_branch(repo): if repo in BASE_BRANCH_MAP: return BASE_BRANCH_MAP[repo] else: From 6f9cbad03eed4db1ad3171bfc4868886b13e9164 Mon Sep 17 00:00:00 2001 From: Joe Sislow Date: Fri, 12 Jan 2018 19:13:10 -0600 Subject: [PATCH 5/7] Update update_dictionary_dependency_chain.py Update defaults in BASE_BRANCH_MAP, removing arbitrary pins --- bin/update_dictionary_dependency_chain.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/update_dictionary_dependency_chain.py b/bin/update_dictionary_dependency_chain.py index 5c718d0..a95cc94 100644 --- a/bin/update_dictionary_dependency_chain.py +++ b/bin/update_dictionary_dependency_chain.py @@ -91,8 +91,8 @@ # edit this if you want to change each branch manually, otherwise # just use the command line arg default_base_branch BASE_BRANCH_MAP = { - 'gdcdatamodel': 'a31edc57056eaf7d71433773d4f030ad0303c1e3', - 'gdcapi': '4ce0f783a736cc8021cb676aaf809e0cfff621cb', + 'gdcdatamodel': 'origin/master', + 'gdcapi': 'origin/master', 'zugs': 'origin/develop', 'esbuild': 'origin/develop', 'runners': 'origin/develop', From c8d248569a9824b6ab3382425f26eec6aec904a9 Mon Sep 17 00:00:00 2001 From: Justin Barnowski Date: Wed, 21 Feb 2018 10:57:12 -0600 Subject: [PATCH 6/7] Removed auto-qa as a target. Added comments for new args --- bin/update_dictionary_dependency_chain.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/bin/update_dictionary_dependency_chain.py b/bin/update_dictionary_dependency_chain.py index a95cc94..1d12618 100644 --- a/bin/update_dictionary_dependency_chain.py +++ b/bin/update_dictionary_dependency_chain.py @@ -24,12 +24,15 @@ in step 1.5. Go fight travis. First(1) step +Run `python update_dictionary_dependency_chain.py --help` to get more info ```bash python update_dictionary_dependency_chain.py \ # --target datamodel \ # only update datamodel --branch chore/bump-deps \ # push on this branch - --dictionary_commit SHA1 # change to this dictionary commit + --dictionary_commit SHA1 \ # change to this dictionary commit + --open_browser \ # Optional: open repo in browser after commit + --dry_run # Optional: dry run, simply create branches, do not push them ``` Second(2) step @@ -39,7 +42,9 @@ --target downstream \ # don't update datamodel --branch chore/bump-deps \ # push on this branch --dictionary_commit SHA1 \ # change to this dictionary commit - --datamodel_commit SHA1 # change to this datamodel commit + --datamodel_commit SHA1 \ # change to this datamodel commit + --open_browser \ # Optional: open repo in browser after commit + --dry_run # Optional: dry run, simply create branches, do not push them ``` Note: you can set the OPEN_CMD environment variable to a browser to @@ -71,7 +76,6 @@ 'zugs': ['setup.py'], 'esbuild': ['requirements.txt'], 'runners': ['setup.py'], - 'auto-qa': ['requirements.txt'], 'authorization': ['auth_server/requirements.txt'], 'legacy-import': ['setup.py'] } @@ -83,7 +87,6 @@ 'zugs': 'git@github.com:NCI-GDC/zugs.git', 'esbuild': 'git@github.com:NCI-GDC/esbuild.git', 'runners': 'git@github.com:NCI-GDC/runners.git', - 'auto-qa': 'git@github.com:NCI-GDC/auto-qa.git', 'authorization': 'git@github.com:NCI-GDC/authorization.git', 'legacy-import': 'git@github.com:NCI-GDC/legacy-import.git', } From 43307401568696d81fabd103160f71af3580cd64 Mon Sep 17 00:00:00 2001 From: Justin Barnowski Date: Wed, 21 Feb 2018 11:03:25 -0600 Subject: [PATCH 7/7] fix(travis): Looks like we need postgresql version specified --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index 91867df..8c2a262 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,10 @@ language: python python: - "2.7" + +addons: + postgresql: "9.4" + install: - python setup.py install - pip install -r dev-requirements.txt -r requirements.txt