Merge pull request #30 from NathanWorkman/dev

Upgrade dependencies, fix docker setup, cleanup frontend.

.editorconfig

@@ -0,0 +1,29 @@
+# EditorConfig helps developers define and maintain consistent
+# coding styles between different editors and IDEs
+# https://editorconfig.org
+root = true
+
+[*]
+end_of_line = lf
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+[*.{md,rst,txt}]
+indent_style = space
+indent_size = 4
+
+[*.{ini,toml}]
+indent_style = space
+indent_size = 4
+
+[*.{js,json,yml,yaml}]
+indent_style = space
+indent_size = 2
+
+[*.{css,py,scss,html}]
+indent_size = 4
+indent_style = space
+
+[*.md]
+trim_trailing_whitespace = false

.gitignore

@@ -1,7 +1,7 @@
 
 # general things to ignore
 build/
-dist/
+# dist/
 *.egg-info/
 *.egg
 *.py[cod]
@@ -23,15 +23,34 @@ __pycache__
 /node_modules
 
 # project dist static files
-/static/
-/media/
+/static
+media
+/finder/static/assets
 
+.vscode
 # project secrets
 .env
 
 # due to ansible logs
 *.log
 *.retry
 
-# local db backups
-backups/
+# background process stuff
+nohup.out
+
+# Ignore data dumps
+fixtures/
+initial_data.dump
+
+# ignore scraper things
+# Google Scraper specific
+Drivers/chromedriver
+Drivers/geckodriver
+Private/
+keywords.txt
+out.csv
+# cache for GoogleScraper testings
+.scrapecache/
+*.db
+/results
+*.log

Dockerfile

@@ -0,0 +1,18 @@
+#!/bin/bash
+FROM python:3.6.10-buster
+LABEL Nathan Workman <nathancworkman@gmail.com>
+
+ENV PYTHONDONTWRITEBYTECODE 1
+ENV PYTHONUNBUFFERED 1
+
+RUN apt-get update && apt-get -y upgrade && apt-get install -y sudo
+RUN apt-get install -y git apt-utils build-essential
+
+RUN mkdir -p /usr/src/app
+WORKDIR /usr/src/app
+COPY . .
+
+RUN pip install --upgrade pip
+RUN pip install --no-cache-dir -r requirements.txt
+
+EXPOSE 8000

Makefile

@@ -2,119 +2,64 @@ PROJECT_NAME = seeker
 SHELL := /bin/sh
 help:
 	@echo "Please use 'make <target>' where <target> is one of"
-	@echo " reset                                Complete reset of database and migrations (Be Careful)"
-	@echo " migrate                              Run Django migrations"
-	@echo " migrations                           Create Django migrations"
-	@echo " user                                 Create user account"
-	@echo " clean                                Remove all *.pyc, .DS_Store and temp files from the project"
-	@echo " collectstatic                        Collect static assets"
-	@echo " run                                  Run Django Server"
-	@echo " crawl_spider <spidername>            Run Scrapy Spider"
-	@echo " crawl                                Run ALL Scrapy Spiders"
-	@echo " dump                                 Create database dump/backup"
+	@echo "migrate                   	Run database migrations"
+	@echo "collectstatic				Collect static assets"
+
 
 .PHONY: requirements
 
 
 # Command variables
+COMPOSE = docker-compose
+COMPOSE_CMD = exec app python manage.py
 MANAGE_CMD = python manage.py
-PIP_INSTALL_CMD = pip install
-PLAYBOOK = ansible-playbook
-VIRTUALENV_NAME = venv
-APP_NAME = seeker
+
 # Helper functions to display messagse
 ECHO_BLUE = @echo "\033[33;34m $1\033[0m"
 ECHO_RED = @echo "\033[33;31m $1\033[0m"
 ECHO_GREEN = @echo "\033[33;32m $1\033[0m"
-DATE= `date +'%m_%d_%y'`
+
 # The default server host local development
 HOST ?= localhost:8000
 
-reset: dropdb createdb migrations migrate user run
-
 migrate:
-# Run django migrations
+# Run django migrations inside docker
 	$(call ECHO_GREEN, Running migrations... )
 	( \
-		$(MANAGE_CMD) migrate; \
-	)
-
-user:
-# Create user account
-	$(call ECHO_GREEN, Creating super user... )
-	( \
-		echo "from django.contrib.auth.models import User; User.objects.create_superuser('admin', 'admin@email.com', 'pass')" | ./manage.py shell; \
-	)
-
-test:
-# Run the test cases
-	( \
-		$(MANAGE_CMD) test; \
-	)
-
-clean:
-# Remove all *.pyc, .DS_Store and temp files from the project
-	$(call ECHO_BLUE,removing .pyc files...)
-	@find . -name '*.pyc' -exec rm -f {} \;
-	$(call ECHO_BLUE,removing static files...)
-	@rm -rf $(PROJECT_NAME)/_static/
-	$(call ECHO_BLUE,removing temp files...)
-	@rm -rf $(PROJECT_NAME)/_tmp/
-	$(call ECHO_BLUE,removing .DS_Store files...)
-	@find . -name '.DS_Store' -exec rm {} \;
-
-shell:
-# Run a local shell for debugging
-	( \
-		$(MANAGE_CMD) shell_plus; \
-	)
-
-migrations:
-# Create database migrations
-	( \
-		$(MANAGE_CMD) makemigrations; \
+		$(COMPOSE) $(COMPOSE_CMD) migrate --no-input; \
 	)
 
 collectstatic:
-# Collect static assets
-	$(call ECHO_GREEN, Collecting static assets...)
+# Run django migrations inside docker
+	$(call ECHO_GREEN, Collect static assets... )
 	( \
-		$(MANAGE_CMD) collectstatic; \
+		$(COMPOSE) $(COMPOSE_CMD) collectstatic --no-input; \
 	)
 
-run:
-# run django server
-	$(call ECHO_GREEN, Starting Django Server...)
+migrations:
+# Run django migrations inside docker
+	$(call ECHO_GREEN, Create new database migrations... )
 	( \
-		$(MANAGE_CMD) runserver 0.0.0.0:8300 ; \
-	)
-
-
-crawl:
-# Run ALL scrapy spiders
-	$(call ECHO_GREEN, Running spiders... )
-	 (\
-		python seeker/crawl.py;  \
-	)
-
-crawl_spider:
-# Run scrapy spider
-	$(call ECHO_GREEN, Running $(spider) spider... )
-	 (\
-		scrapy crawl $(spider);  \
+		$(COMPOSE) $(COMPOSE_CMD) makemigrations; \
 	)
 
-createdb:
+build:
+# Run django migrations inside docker
+	$(call ECHO_GREEN, Building new images... )
 	( \
-		createdb -p 5433 $(APP_NAME); \
+		$(COMPOSE) build; \
 	)
 
-dropdb:
+down:
+# Run django migrations inside docker
+	$(call ECHO_GREEN, Tear down containers... )
 	( \
-		dropdb -p 5433 $(APP_NAME);\
+		$(COMPOSE) down; \
 	)
 
-dump:
+up:
+# Run django migrations inside docker
+	$(call ECHO_GREEN, Launching starship... )
 	( \
-		pg_dump -Fc seeker -p 5433 > backups/data_dump_$(DATE).dump; \
+		$(COMPOSE) up; \
 	)

README.md

@@ -3,16 +3,15 @@
     <h1>Seeker</h1>
 </div>
 
-[![Build Status](https://travis-ci.org/NathanWorkman/seeker.svg?branch=master)](https://travis-ci.org/NathanWorkman/seeker)
+# Seeker
 
-## What is Seeker?
 Seeker is just another job board aggregator. Check multiple job boards for positions you might be interested in and organize them all in one convenient location.
 
 ## Setup
 
 ### Docker
 
-Docker installation should be fairly straight forward.
+The easiest way to get started is to clone this repo and run docker-compose:
 
 ```
 git clone git@github.com:NathanWorkman/seeker.git
@@ -21,91 +20,18 @@ docker-compose build
 docker-compose up
 ```
 
-### The hard(er) way:
-
-You will need `yarn` and `virtualenv` installed on your machine.
-
-Install Yarn
-```
-brew install yarn
-```
-
-Install virtualenv
-```
-pip install virtualenv
-```
-
-### To run the project
-```
-git clone git@github.com:NathanWorkman/seeker.git
-cd seeker/
-virtualenv venv -p python3
-source venv/bin/activate
-pip install -r requirements.txt
-yarn
-cd seeker/
-python manage.py migrate
-python manage.py createsuperuser
-make build
-make run
-```
-
 ### To run the spiders
-From the root directory of the `seeker` project `cd` to the seeker app directory.
 
-```
-cd seeker
-```
-and then run the following to run the individual spiders, replacing `spidername` with the spider you wish to run.
+Execute the individual spiders from inside the docker container
 
-```
-scrapy crawl spidername
+```shell
+docker exec -it seeker_app scrapy crawl spidername
 ```
 
 or run all the spiders at once:
 
+```shell
+docker exec -it seeker_app python crawl.py
 ```
-python crawl.py
-```
-
-Navigate to the django admin to view your results.
-
-## TODO
-
-### Boards 
-- [x] DjangoGigs
-- [ ] Indeed
-- [ ] PythonOrg
-- [ ] RemotePython
-- [ ] Stack Overflow
-- [ ] Workable
-- [ ] Lever
-- [ ] Recruiter Box
-- [ ] Greenhouse
-- [ ] TBD
-- [ ] https://hireremote.io
-- [ ] https://weworkremotely.com
-- [ ] https://remoteok.io/
-- [ ] https://jobspresso.co/
-- [ ] https://remotive.io/find-a-remote-job/#s=1
-- [ ] https://www.workingnomads.co/jobs
-- [ ] https://www.mikesremotelist.com/
-- [ ] https://stackoverflow.com/jobs
-- [ ] https://bestremotejob.com/
-- [ ] https://justremote.co/
-- [ ] https://whoishiring.io/
-- [ ] https://www.techjobsforgood.com/?search=remote
-- [ ] https://rmtwrk.com/
-- [ ] https://remotepath.io/
-- [ ] https://www.hiringremote.ly/
-- [ ] https://www.honestlance.com/
-
-
 
-## Made Possible By
-- [Django](https://www.djangoproject.com/)
-- [Scrapy](https://scrapy.org/)
-- [jQuery](https://jquery.com/)
-- Icons - [Devicon](http://konpa.github.io/devicon/)
-- Admin Theme - [Django Suit](https://github.com/darklow/django-suit)
-- NLP - [SpaCy](https://spacy.io/)
+Navigate to `0.0.0.0:8000` to view results.