gp_mkpkg

#!/usr/bin/python

# make_src
# This is part of the Mer git-packaging suite
# It takes a _src file and recreates any tarballs referenced in it.
#
# 3 types:
# native: extract the tarball using git archive <tag>
# upstream git: additionally use git format-patch to make patches
# pristine tar: use ptar and then git format-patch
# upstream with released tarball: as pristine tar but skip the commit
#   labelled "pristine-tar" in the mer-<version> branch

import re
import os
import subprocess
import argparse
import sys
import collections
import yaml
import tempfile
import BlockDumper

# Should come from ~/.vendor and/or commandline option
vendor="mer"
pkg_branch="pkg-%s" % vendor

# Allow overriding of "git' command via the "GIT" environment variable
GIT_CMD = os.environ.get('GIT', 'git')


# Data passing class
class Patches():
    def __init__(self):
        self.spec_Patches=[]
        self.spec_patches=[]
        self.yaml_Patches=[]

class UnknownFormat(Exception):
    pass

def git(*args, **kwargs):
    """Run git with arguments, return stdout as string"""
    return subprocess.check_output([GIT_CMD] + list(args), **kwargs)

def log(msg):
    if args.verbose:
	print(msg)

def fail(msg, *args):
    """Report an error message and exit with failure"""
    print >>sys.stderr, 'ERROR:', msg % args
    sys.exit(1)

def checkout_packaging_branch(pkg, force):
    """
    Ensure we're on the latest packaging branch commit.
    If force is False and we're on the packaging branch then no-op
    (this essentially lets us run with uncommited changes to _src,
    spec etc so the patch-sets can be added.)
    """
    if force:
        git('checkout', '-f', pkg)

    try:
        branch = git('symbolic-ref', 'HEAD').replace('refs/heads/', '').rstrip()
    except subprocess.CalledProcessError:
        return # We're not on a symbolic ref; assume tag

    if not force and branch != pkg:
        fail('Must be on the %s branch to use --no-checkout', pkg)

def guess_format(archive):
    if (re.search(r'\.t(ar\.)?gz$', archive)):
        return ("tar", "gzip")
    elif (re.search(r'\.tar\.bz2$', archive)):
        return ("tar", "bzip2")
    elif (re.search(r'\.tar\.xz$', archive)):
        return ("tar", "xz")
    else:
        raise UnknownFormat()

def count_commits(a, b):
    return len(git('rev-list', '..'.join((a, b)), '--').splitlines())


def prepare_tarball_and_patches(args):
    p = Patches()

    src_f = open("_src")

    for src in src_f:
        src = src.rstrip()
        src = src.split(":")
        vcs = src[0]
        tag1, tag2 = None, None

        if vcs.startswith("#"):
            continue

        if (vcs == "git" ):
            # git:<tarball>:<tag1>:<tag2>
            # make a tarball at tag1 and patches to tag2 (if present)
            if len(src) < 3:
                raise Exception("Not enough arguments for 'git' line in _src")

            tarball, tag1 = src[1:3]
            if len(src) > 3:
                tag2 = src[3]

            if args.make_tarball:
                # Determine archive/compression
                try:
                    format, compression = guess_format(tarball)
                    # Make a suitable tarball
                    subprocess.call(
                        GIT_CMD + " archive --format=%(format)s --prefix=src/ %(tag1)s "
                        "| %(compression)s > %(tarball)s" % locals(),
                        shell=True)
                except UnknownFormat :
                    fail("Can't determine format of tarball %s", tarball)

        elif (vcs == "pristine-tar"):
            # pristine-tar:<tarball>:<tag1>:<tag2>
            # extract upstream tarball and make patches from tag1 to tag2
            # (if present) skipping the special 'pristine-tar' diff patch
            if len(src) < 3:
                raise Exception("Not enough (<3) arguments for 'pristine-tar' line in _src: '%s'" % (src))

            tarball, tag1 = src[1:3]
            if len(src) > 3:
                tag2 = src[3]
                # For pristine tar style repos the first commit is the
                # pristine tar delta stored in a commit labeled pristine-tar-delta
                # Skip the first commit using commit~N terminology
                tag1 = "%s~%d" %(tag2, count_commits(tag1, tag2) - 1)
            else:
                # We don't have any patches so only drop the last commit.
                tag1 = "%s~1" % (tag1)

            if args.make_tarball:
                subprocess.call(
                    [ "/usr/bin/pristine-tar", "checkout", tarball],
                    shell=False)

            # Prevent pristine-tar from altering the %setup line
            args.setup_src = False

        else:
            raise Exception("_src storage %s not recognised" % vcs)

        # If a tag range was specified, create patches
        # -M is --find-renames but that's not available on suse 11.4
        if tag2:
            patches = git('format-patch', '-M', '--binary', '--no-renames', '--ignore-submodules',
                    '-N', '--no-numbered', '..'.join((tag1, tag2)), '--').splitlines()

            count = 1
            for patch in patches:
                p.spec_Patches.append("Patch%(count)d:\t%(patch)s\n" % locals())
                count+=1
                if not args.make_tarball:
                    os.remove(patch)

            count = 1
            for patch in patches:
                p.spec_patches.append("%%patch%(count)d -p1\n" % locals())
                count+=1

            # p.yaml_Patches.append("Patches:\n")
            for patch in patches:
                #p.yaml_Patches.append("- %(patch)s\n" % locals())
                p.yaml_Patches.append(patch)

        else:
            log("No patches requested")

    return p

def apply_patches_to_spec(args, p, specfilename):

    spec_f = open(specfilename, "r+")
    spec_new = []

    Source_match = re.compile(r"^Source")
    Patch_match = re.compile(r"^Patch")
    prep_match = re.compile(r"^%prep")
    patch_match = re.compile(r"^%patch")

    # Skip to the SourceXX: line
    line = "python has no do: while syntax"
    while line:
        line = spec_f.readline()
        if not Patch_match.search(line):
            spec_new.extend(line)
        if not Source_match.search(line):
            continue
        break

    # Now print all Source lines but not Patch lines
    while line:
        line = spec_f.readline()
        if Patch_match.search(line):
            continue
        if Source_match.search(line):
            spec_new.extend(line)
            continue
        next_line = line
        break

    spec_new.extend(p.spec_Patches)
    spec_new.extend(next_line)

    # Skip to %prep
    while line:
        line = spec_f.readline()
        if not Patch_match.search(line):
            spec_new.extend(line)
        if prep_match.search(line):
            break

    # Any remaining spec content, skipping %patch lines
    # and updating when %setup is found
    done_setup = False
    while line:
        line = spec_f.readline()
        if patch_match.search(line):
            continue

        if re.search(r'^%setup', line):
            if done_setup:
                print "WARNING: multiple %setups found - please check patches are in the right place in spec"
            else:
                done_setup = True
                if args.setup_src and not re.search(r'-n src(/s+.)?', line):
                    print "Adjusting %setup since git-pkg unpacks to src/ and there's no -n src in %setup"
                    spec_new.extend("# Adjusting %%setup since git-pkg unpacks to src/\n")
                    spec_new.extend("# %s" % re.sub(r'%', '%%', line))
                    print line.strip()
                    line,c = re.subn(r'-n\s*\S*', r'-n src', line)
                    if c == 0:
                        line = line.strip() + " -n src\n"
                     
                    print "converted to"
                    print line.strip()
                # Now output modified or orig %setup and any patches
                spec_new.extend(line)
                spec_new.extend(p.spec_patches)
                continue

        spec_new.extend(line)

    spec_f.seek(0)
    spec_f.writelines(spec_new)
    spec_f.close()

def apply_patches_to_yaml(args, p, yamlfilename):

    f = open(yamlfilename, "r+")
    specify = yaml.load(f)

    specify['Patches'] = p.yaml_Patches
    if args.setup_src:
        specify['SetupOptions'] = '-q -n src'
    f.seek(0)
    f.truncate()
    yaml.dump(specify, f, Dumper=BlockDumper.BlockDumper, default_flow_style=False)
    f.close()

    log("Running specify")
    subprocess.check_output(['specify', "-N", "-s", "-n"])

def apply_patches(args, p):
    ""
    for f in os.listdir("."):
        if f.endswith(".yaml"):
            apply_patches_to_yaml(args, p, f)
            # Could run specify here - fn name would change
            return
        if f.endswith(".spec"):
            spec = f
    # No yaml found, patch the spec
    apply_patches_to_spec(args, p, spec)


parser = argparse.ArgumentParser(description='gitpkg build preparation')

parser.add_argument('-b', '--build',
                    dest='build', action='store_true',
                    help="Just prepare tarballs/patches for building - don't modify spec/yaml from git")

parser.add_argument('-n', '--no-checkout',
                    dest='force_checkout', action='store_false',
                    help="Don't force checkout the latest pkg branch (useful if working on packaging)")

parser.add_argument('--no-src',
                    dest='setup_src', action='store_false',
                    help="Don't modify %setup to use -n src/ (useful for pristine tar")

parser.add_argument('--no-tarball',
                    dest='make_tarball', action='store_false',
                    help="Don't create the tarball/patches")

parser.add_argument('--git-dir',
                    help="Specify the directory holding the gitpkg repo")

parser.add_argument('tag', metavar='tag', type=str, nargs='?',
                    default=pkg_branch,
                    help='A tag to force checkout')

parser.add_argument('-V', '--verbose',
                    dest='verbose', action='store_true',
                    help="Verbose output.")

args = parser.parse_args()

if args.git_dir:
    try:
        with open(os.devnull, 'w') as null:
            git("--git-dir", args.git_dir, "rev-parse", "--git-dir", stderr=null)
    except subprocess.CalledProcessError:
        try:
            # not a git dir - try appending .git
            with open(os.devnull, 'w') as null:
                git("--git-dir", os.path.join(args.git_dir,".git"), "rev-parse", "--git-dir", stderr=null)
            args.git_dir = os.path.join(args.git_dir,".git")
        except subprocess.CalledProcessError:
            fail("The --git-dir option must point to a git repo.\n"
                 "%s nor %s/.git are a git directory" % (args.git_dir, args.git_dir))
    os.environ['GIT_DIR']=args.git_dir
    os.environ['GIT_WORK_TREE']=os.getcwd()
    os.environ['GIT_INDEX_FILE']=tempfile.NamedTemporaryFile(prefix="gp_index_", delete=False).name
    git("read-tree", "--empty")

# Checkout the tag (default is the hardcoded pkg_branch)
checkout_packaging_branch(args.tag, args.force_checkout)
patches=prepare_tarball_and_patches(args)
if not args.build:
    apply_patches(args, patches)

if args.git_dir:
    os.remove(os.environ['GIT_INDEX_FILE'])