-
Notifications
You must be signed in to change notification settings - Fork 18
/
setup.py
147 lines (129 loc) · 5.49 KB
/
setup.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
#-------------------------------------------------------------------------------
# Author: Lukasz Janyst <lukasz@jany.st>
# Date: 25.11.2017
#
# Licensed under the 3-Clause BSD License, see the LICENSE file for details.
#-------------------------------------------------------------------------------
import json
import sys
import os
from setuptools.command.build_py import build_py
from distutils.spawn import find_executable
from setuptools import setup
from subprocess import Popen
from scrapy_do import __version__
from shutil import copyfile, rmtree
#-------------------------------------------------------------------------------
# Package description
#-------------------------------------------------------------------------------
with open('README.rst') as readme:
long_description = readme.read()
#-------------------------------------------------------------------------------
# Run command
#-------------------------------------------------------------------------------
def run_command(args, cwd):
p = Popen(args, cwd=cwd)
p.wait()
return p.returncode
#-------------------------------------------------------------------------------
# Build the React web app
#-------------------------------------------------------------------------------
class build_ui(build_py):
def run(self):
if not self.dry_run:
#-------------------------------------------------------------------
# Check and set the environment up
#-------------------------------------------------------------------
target_dir = os.path.join(self.build_lib, 'scrapy_do', 'ui')
if os.path.exists(target_dir):
rmtree(target_dir)
ui_path = os.path.join(os.getcwd(), 'ui')
if not os.path.exists(ui_path):
print('[!] The ui directory does not exist')
sys.exit(1)
npm = find_executable('npm')
if npm is None:
print('[!] You need to have node installed to build this app')
sys.exit(1)
#-------------------------------------------------------------------
# Build the JavaScript code
#-------------------------------------------------------------------
ret = run_command([npm, 'install'], ui_path)
if ret != 0:
print('[!] Installation of JavaScript dependencies failed')
sys.exit(1)
ret = run_command([npm, 'run-script', 'build'], ui_path)
if ret != 0:
print('[!] Build of JavaScript artefacts failed')
sys.exit(1)
#-------------------------------------------------------------------
# Create a list of artefacts
#-------------------------------------------------------------------
artefacts = [
'asset-manifest.json',
'favicon.png',
'scrapy-do-logo.png',
'index.html',
'manifest.json'
]
build_dir = 'ui/build'
asset_manifest = os.path.join(build_dir, artefacts[0])
if not os.path.exists(asset_manifest):
print('[!] Asset manifest does not exist.')
sys.exit(1)
assets = json.loads(open(asset_manifest, 'r').read())
for _, asset in assets['files'].items():
artefacts.append(asset)
#-------------------------------------------------------------------
# Copy the artefacts to the dist root
#-------------------------------------------------------------------
print('Copying JavaScript artefacts to', target_dir)
for artefact in artefacts:
if artefact.startswith('/'):
artefact = artefact[1:]
source_file = os.path.join(build_dir, artefact)
target_file = os.path.join(target_dir, artefact)
target_prefix = os.path.dirname(target_file)
if not os.path.exists(target_prefix):
os.makedirs(target_prefix)
copyfile(source_file, target_file)
build_py.run(self)
#-------------------------------------------------------------------------------
# Setup
#-------------------------------------------------------------------------------
setup(
name = 'scrapy-do',
version = __version__,
author = 'Lukasz Janyst',
author_email = 'xyz@jany.st',
url = 'https://jany.st/scrapy-do.html',
description = 'A daemon for scheduling Scrapy spiders',
long_description = long_description,
license = 'BSD License',
packages = ['scrapy_do', 'scrapy_do.client'],
include_package_data = True,
cmdclass={
'build_py': build_ui
},
package_data={
'': ['*.conf'],
},
scripts=['scrapy-do', 'scrapy-do-cl'],
classifiers = [
'Framework :: Scrapy',
'Development Status :: 4 - Beta',
'Intended Audience :: Developers',
'License :: OSI Approved :: BSD License',
'Operating System :: OS Independent',
'Programming Language :: Python',
'Programming Language :: Python :: 3',
'Programming Language :: Python :: 3.7',
'Topic :: Internet :: WWW/HTTP',
'Environment :: Console',
'Environment :: No Input/Output (Daemon)',
],
install_requires = [
'scrapy', 'twisted', 'pyOpenSSL', 'psutil', 'python-dateutil',
'schedule', 'pem', 'tabulate', 'requests', 'autobahn', 'tzlocal'
]
)