Skip to content

Commit

Permalink
github v2.2 Buildon framework for python3.x
Browse files Browse the repository at this point in the history
  • Loading branch information
@JeyanthiBalakrishnan
JeyanthiBalakrishnan committed Jun 29, 2018
1 parent 7ed33df commit 008238f
Show file tree
Hide file tree
Showing 3 changed files with 112 additions and 116 deletions.
19 changes: 9 additions & 10 deletions scripts/python/src/app.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,28 +60,27 @@ def upload():
if file and allowed_file(file.filename):
# Make the filename safe, remove unsupported chars
filename = file.filename
file_temp = filename.split(".log")[0]
file_temp = filename.split(".log")[0]
commitid = file_temp[-7:]
print "COMMIT: " + commitid
print ("COMMIT: " + commitid)


##Uplaod folder for Jenkins and Kube logs
uploadPath=logbasepath
if not os.path.exists(uploadPath+commitid):
os.makedirs(uploadPath+commitid)
uploadPath=logbasepath
if not os.path.exists(uploadPath+commitid):
os.makedirs(uploadPath+commitid)
file.save(os.path.join(uploadPath+commitid, filename))
return 'ok'

@app.route('/setup', methods=['POST'])
def setup():
print request.data
print (request.data)
build = BuildOn(request.data)
buildon = build.setup(request.data)
print buildon
print (buildon)
if (buildon != 0):
##To forward logs to ElasticSearch. If required enable the below line and copy this script from backup folder
#pid = subprocess.Popen([sys.executable, "/home/ubuntu/BuildOn/LogsForwarding.py"])
##To update jenkins job's live status in root.reports table
pid = subprocess.Popen([sys.executable, "/home/ubuntu/BuildOn/dbupdate.py"])
return "ok"

pid = subprocess.Popen([sys.executable, "/home/ubuntu/BuildOn/dbupdate.py"])
return "ok"
155 changes: 73 additions & 82 deletions scripts/python/src/buildon.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,16 +34,16 @@
class BuildOn(object):
def __init__(self, data):
##Fetch server details and ports from properties file
print data
data = json.loads(data)
print (data)
data = json.loads(data.decode('utf-8'))
config = configparser.ConfigParser()
config.sections()
config.read('buildon.properties')
config.sections()
'server' in config
self.host_ip=config['server']['kubemaster_ip']
self.k8s_namespace=config['server']['k8s_namespace']
self.k8s_serviceaccount=config['server']['k8s_serviceaccount']
self.k8s_namespace=config['server']['k8s_namespace']
self.k8s_serviceaccount=config['server']['k8s_serviceaccount']
self.upload_port=config['server']['framework_port']
self.gitlabtoken=config['server']['gitlab_token']
self.pghost=config['server']['pghost']
Expand All @@ -57,16 +57,16 @@ def __init__(self, data):
self.bitbucket_port=config['server']['bitbucket_port']
self.bitbucket_token=config['server']['bitbucket_token']
self.bitbucket_serviceaccount=config['server']['bitbucket_serviceaccount']
self.CustomMavenSettings=config['server']['CustomMavenSettings']
self.CustomMavenSettings=config['server']['CustomMavenSettings']
self.SettingsXmlPath=config['server']['SettingsXmlPath']

print "Printing Payload Data ......"
print data
print ("Printing Payload Data ......")
print (data)

##Extract Payload data from respective Source Code Repository.Here "head_commit" is the key word
#which differs between GitLab and GitHub and eventKey is to identify BitBucket's Payload content.Using this payload extraction is done.
if 'eventKey' in data:
print "BitBucket Commit"
print ("BitBucket Commit")
self.commitid = format(data['changes'][0]['toHash'])[0:7]
self.unique = format(data['changes'][0]['ref']['id'])
self.scmuser = format(data['actor']['emailAddress'])
Expand All @@ -80,18 +80,18 @@ def __init__(self, data):
self.jenkinsfileURL="http://"+self.bitbucket_ip+":"+self.bitbucket_port+"/projects/"+self.projectKey+"/repos/"+self.repo_name+"/raw/Jenkinsfile"
self.httpurl='http://'+self.bitbucket_serviceaccount+'@'+self.bitbucket_ip+':'+self.bitbucket_port+'/scm/'+self.projectKey+'/'+self.repo_name+'.git'
self.SCM = 'bitbucket'
print "Printing Paylaod data...***"
print "CommitID..."+ self.commitid
print "UID..."+ self.uid
print "Branch..." + self.tuid
print "repo_name..."+ self.repo_name
print "buildpath" + self.buildpath
print "scmuser..."+ self.scmuser
print "webhookcheck..." + self.webhookcheck
print "jenkisnfileURL..." + self.jenkinsfileURL
print ("Printing Paylaod data...***")
print ("CommitID..."+ self.commitid)
print ("UID..."+ self.uid)
print ("Branch..." + self.tuid)
print ("repo_name..."+ self.repo_name)
print ("buildpath" + self.buildpath)
print ("scmuser..."+ self.scmuser)
print ("webhookcheck..." + self.webhookcheck)
print ("jenkisnfileURL..." + self.jenkinsfileURL)

elif 'head_commit' not in data:
print "GitLab Commit"
print ("GitLab Commit")
self.commitid = format(data['commits'][0]['id'])[0:7]
self.unique = format(data['ref'])
self.tuid = self.unique.split("/")[2]
Expand All @@ -106,21 +106,21 @@ def __init__(self, data):
self.jenkinsfileURL=self.repohttpurl + "/" + "raw" + "/" + self.tuid + "/" + "Jenkinsfile?private_token="+self.gitlabtoken
self.SCM = 'gitlab'

print "Printing Paylaod data...***"
print "CommitID..."+ self.commitid
print "UID..."+ self.uid
print "Branch..." + self.tuid
print "repo_name..."+ self.repo_name
print "buildpath" + self.buildpath
print "scmuser..."+ self.scmuser
print "webhookcheck..." + self.webhookcheck
print "httpurl ..." + self.httpurl
print "repohttpurl..." + self.repohttpurl
print "jenkisnfileURL..." + self.jenkinsfileURL
print ("Printing Paylaod data...***")
print ("CommitID..."+ self.commitid)
print ("UID..."+ self.uid)
print ("Branch..." + self.tuid)
print ("repo_name..."+ self.repo_name)
print ("buildpath" + self.buildpath)
print ("scmuser..."+ self.scmuser)
print ("webhookcheck..." + self.webhookcheck)
print ("httpurl ..." + self.httpurl)
print ("repohttpurl..." + self.repohttpurl)
print ("jenkisnfileURL..." + self.jenkinsfileURL)
else:
print "GitHub commit"
print "Printing data"
print data
print ("GitHub commit")
print ("Printing data")
print (data)
self.commitid = format(data['commits'][0]['id'])[0:7]
self.unique = format(data['ref'])
self.tuid = self.unique.split("/")[2]
Expand All @@ -135,18 +135,18 @@ def __init__(self, data):
self.repohttpurl = self.httpurl.replace(".git","")
self.jenkinsfileURL="https://raw.githubusercontent.com/"+self.repo_fullName + "/" + self.tuid + "/" + "Jenkinsfile"
self.SCM = 'github'
print "Printing Paylaod data...***"
print "CommitID..."+ self.commitid
print "UID..."+ self.uid
print "Branch..." + self.tuid
print "repo_name..."+ self.repo_name
print "repo_fullname..."+ self.repo_fullName
print "buildpath" + self.buildpath
print "scmuser..."+ self.scmuser
print "webhookcheck..." + self.webhookcheck
print "httpurl ..." + self.httpurl
print "repohttpurl..." + self.repohttpurl
print "jenkisnfileURL..." + self.jenkinsfileURL
print ("Printing Paylaod data...***")
print ("CommitID..."+ self.commitid)
print ("UID..."+ self.uid)
print ("Branch..." + self.tuid)
print ("repo_name..."+ self.repo_name)
print ("repo_fullname..."+ self.repo_fullName)
print ("buildpath" + self.buildpath)
print ("scmuser..."+ self.scmuser)
print ("webhookcheck..." + self.webhookcheck)
print ("httpurl ..." + self.httpurl)
print ("repohttpurl..." + self.repohttpurl)
print ("jenkisnfileURL..." + self.jenkinsfileURL)
if (self.SCM == "gitlab"):
self.scmServiceAccount = self.gitlabtoken
elif (self.SCM == "bitbucket"):
Expand All @@ -155,26 +155,29 @@ def __init__(self, data):
self.scmServiceAccount = self.githubtoken

os.environ['commit_id'] = self.commitid
os.environ['uid'] = self.uid
os.environ['pghost'] = self.pghost
os.environ['pgport'] = self.pgport
os.environ['pgdatabase'] = self.pgdatabase
os.environ['pguser'] = self.pguser
os.environ['pgupass'] = self.pgupass
print self.logbasepath
print (self.logbasepath)
os.environ['logbasepath'] = self.logbasepath
os.environ['k8s_namespace'] = self.k8s_namespace
os.environ['k8s_serviceaccount'] = self.k8s_serviceaccount
print "environment variable"
print os.environ['pgupass']
print os.environ['logbasepath']
os.environ['k8s_namespace'] = self.k8s_namespace
os.environ['k8s_serviceaccount'] = self.k8s_serviceaccount
os.environ['CustomMavenSettings'] = self.CustomMavenSettings
os.environ['SettingsXmlPath'] = self.SettingsXmlPath
print ("environment variable")
print (os.environ['pgupass'])
print (os.environ['logbasepath'])
def dbinsert(self):
print "Inside self.dbinsert"
print ("Inside self.dbinsert")
now = datetime.datetime.now()
self.start_timestamp = now.strftime("%Y-%m-%d %H:%M:%S")
self.start_date = now.strftime("%Y-%m-%d")
print self.scmuser
print self.tuid
print self.repo_name
print (self.scmuser)
print (self.tuid)
print (self.repo_name)
result = ''
try:
##Check if WebHook is ON/OFF. If OFF exit the execution.Exit only if commit is from GitLab checkin and Webhook is OFF.
Expand All @@ -184,13 +187,13 @@ def dbinsert(self):
for row in cur:
result = row[0]
except Exception as e:
print e
print 'Preference Error'
print (e)
print ('Preference Error')
finally:
con.close()
print "self.webhookcheck: " + self.webhookcheck
print ("self.webhookcheck: " + self.webhookcheck)
if ( result == 0 and self.webhookcheck != "" and 'BOT' not in self.webhookcheck ): ##Exits without BuildOn flow if webhook is turnedd off when committign code directly to SCM
print 'WebHook is off'
print ('WebHook is off')
return 0

if ( self.webhookcheck != "" ):
Expand All @@ -200,14 +203,14 @@ def dbinsert(self):
trigger_from="FRAMEWORK"
else:
trigger_from="UI"
print trigger_from
print 'trigger_from'
print (trigger_from)
print ('trigger_from')
file = open(self.buildpath+"/Jenkinsfile").read().splitlines()
try:
for index, line in enumerate(file):
if "stage ('" in line or "stage('" in line:
ci_jobname =line.split("'")[1]
print ci_jobname
print (ci_jobname)
con = pg8000.connect(host=self.pghost, port=int(self.pgport), user=self.pguser, password=self.pgupass, database=self.pgdatabase)
cur = con.cursor()
est_dur = ''
Expand All @@ -223,53 +226,41 @@ def dbinsert(self):
con.commit()
con.close()
except Exception as e:
print 'Insert Error Occurs'
print e
print ('Insert Error Occurs')
print (e)

def setup(self, data):
if not os.path.exists(self.buildpath):
os.makedirs(self.buildpath)
try:

if ('github' in self.SCM or 'gitlab' in self.SCM):
print "BitBucket Jenkinsfile"
print ("BitBucket Jenkinsfile")
wget.download(self.jenkinsfileURL, out=self.buildpath+"/Jenkinsfile")
else:
print "BitBucket Jenkinsfile"
print ("BitBucket Jenkinsfile")
os.system('cd '+self.buildpath +'; curl -O -G -H "Authorization: Bearer MTM1MzMwNzI5NjIyOkG16lNX7T2F/ihcShuLIISkKc2P" '+ self.jenkinsfileURL +' -d at='+self.unique)
except Exception as e:
print "Error in downloading files from SCM - Jenkinsfile or fetch image from Jenkinsfile"
print ("Error in downloading files from SCM - Jenkinsfile or fetch image from Jenkinsfile")
try:
file = open(self.buildpath+"/Jenkinsfile").read().splitlines()
for index, line in enumerate(file):
if "env.dockerimagename" in line:
line_temp = line
imagename_temp=line_temp.split("=")[1]
imagename=imagename_temp.strip()
print "Docker Image Name: " + imagename
print ("Docker Image Name: " + imagename)
except Exception as e:
print e
print (e)
try:
##DB insert function call
hook = self.dbinsert()
if ( hook == 0 ):
return 0
#subprocess.Popen(['kubectl run '+ self.uid +' -i --tty --rm --restart=Never --image='+imagename+ ' --env="commitID='+self.uid+',branchName='+self.tuid+',gitURL='+self.httpurl+',KUBEMASTER='+self.host_ip+',UPLOADPORT='+self.upload_port+'"'], shell=True)
subprocess.Popen(['kubectl run '+ self.uid +' -i --tty --rm --restart=Never --image='+imagename+ ' --env="commitID='+self.uid+'" --env="branchName='+self.tuid+'" --env="gitURL='+self.httpurl+'" --env="KUBEMASTER='+self.host_ip+'" --env="UPLOADPORT='+self.upload_port+'" --env="scmServiceAccount='+self.scmServiceAccount+'"'], shell=True)
if(self.CustomMavenSettings == "true" ):
doc_cont = os.popen("curl -k -s `(kubectl config view | grep server | cut -f 2- -d \":\" | tr -d \" \")`/api/v1/namespaces/"+self.k8s_namespace+"/pods/"+self.uid+" --header \"Authorization: Bearer `(kubectl describe secret $(kubectl get secrets | grep "+self.k8s_serviceaccount+" | cut -f1 -d ' ') | grep -E '^token' | cut -f2 -d':' | tr -d '\t')`\"").read()
print doc_cont
cont_json = json.loads(doc_cont)
podstatus = cont_json['status']['phase']
print podstatus
if "Running" in podstatus:
print "inside Running if"
subprocess.Popen(['kubectl cp '+self.SettingsXmlPath+' '+self.k8s_namespace+'/'+ self.uid +':/usr/share/maven/conf/settings.xml'], shell=True)
print "after kubectl cp"
else:
print "settings.xml flag is false or custom settings.xml not found to execute sonar,nexus,tomcat"
except NameError as ne:
print ne
print (ne)
raise BuildOn(
"Error in replacing <appndid> in Jenkinsfile"
)
Expand All @@ -285,4 +276,4 @@ def replacestring (filepath, orgstring, replacestring):
filehandle.write(lines)
filehandle.close()
else:
print('File not found{}'.filepath)
print('File not found{}'.filepath)
Loading

0 comments on commit 008238f

Please sign in to comment.