-
Notifications
You must be signed in to change notification settings - Fork 8
/
past-data-extractor.py
83 lines (68 loc) · 3.45 KB
/
past-data-extractor.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
'''
Akond Rahman
Feb 07, 2018
Script to extract time wise data extract
'''
import os
import shutil
import subprocess
def get_immediate_subdirs(a_dir):
return [name for name in os.listdir(a_dir)
if os.path.isdir(os.path.join(a_dir, name))]
def generatePastData(folder_path, y_p, m_p):
all_dirs = get_immediate_subdirs(folder_path)
for dir_ in all_dirs:
srcFolder = folder_path + dir_ + '/'
for year_ in y_p:
for mont_ in m_p:
folder2create = folder_path + dir_ + '-' + year_ + '-' + mont_ + '/'
print '='*50
print folder2create
print '-'*25
if((os.path.exists(folder2create))==False):
try:
shutil.copytree(srcFolder, folder2create)
'''
now do a reset
'''
cdCommand = "cd " + folder2create + " ; "
date2reset = year_ + '-' + mont_ + '-' + '28' ## 28 th of the month
commitCommand = "git checkout -f `git rev-list -n 1 --before='"+ date2reset +"' master`"
command2Run = cdCommand + commitCommand
try:
subprocess.check_output(['bash','-c', command2Run])
except subprocess.Error as subpro_err:
print 'Subprocess error:', subpro_err
except shutil.Error as err_:
print 'Directory not copied, error:', err_
print '='*50
if __name__=='__main__':
# folder2walk = '/Users/akond/SECU_REPOS/test-pup/'
# y_list = ['2011', '2012', '2013', '2014', '2015', '2016']
# m_list = ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12']
# folder2walk = '/Users/akond/SECU_REPOS/wiki-pupp/'
# y_list = ['2006', '2007', '2008', '2009', '2010','2011', '2012', '2013', '2014', '2015', '2016']
# m_list = ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12']
# folder2walk = '/Users/akond/SECU_REPOS/ostk-pupp/'
# y_list = ['2011', '2012', '2013', '2014', '2015', '2016']
# m_list = ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12']
# folder2walk = '/Users/akond/SECU_REPOS/berg-chef/'
# y_list = [ '2013', '2014', '2015', '2016']
# m_list = ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12']
# folder2walk = '/Users/akond/SECU_REPOS/cdat-chef/'
# y_list = ['2012', '2013', '2014', '2015', '2016']
# m_list = ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12']
# folder2walk = '/Users/akond/SECU_REPOS/clcr-chef/'
# y_list = ['2012', '2013', '2014', '2015', '2016']
# m_list = ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12']
# folder2walk = '/Users/akond/SECU_REPOS/expr-chef/'
# y_list = ['2012', '2013', '2014', '2015', '2016']
# m_list = ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12']
### ICSE 2019 Work
# folder2walk = '/Users/akond/PUPP_REPOS/openstack-downloads/extra_repos_icse19/'
# y_list = ['2018']
# m_list = ['04', '05', '06']
# folder2walk = '/Users/akond/PUPP_REPOS/wikimedia-downloads/extra_repos_icse19/'
# y_list = ['2018']
# m_list = ['04', '05', '06']
generatePastData(folder2walk, y_list, m_list)