Skip to content

Commit

Permalink
Project rename
Browse files Browse the repository at this point in the history
  • Loading branch information
@t3eHawk
t3eHawk committed Dec 7, 2022
1 parent 7d9c78e commit 41a6c77
Show file tree
Hide file tree
Showing 25 changed files with 156 additions and 145 deletions.
6 changes: 3 additions & 3 deletions README.md
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
devoe
PyDin
=====

Devoe is a Python ETL framework that allows you to develop, setup and maintain
PyDin is a Python ETL framework that allows you to develop, setup and maintain
ETL processes.

This module was originally a simple standalone script on a small server to run
Expand All @@ -13,7 +13,7 @@ workarounds.
So we just decided that this script must evolve together with a project and
must become something else.

Devoe follows certain ETL phisolophy that was born in our own expirience.
PyDin follows certain ETL phisolophy that was born in our own expirience.
It was created to work in real production environment with thoushands of
daily jobs.

Expand Down
96 changes: 48 additions & 48 deletions db/oracle.sql
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
/*******************************************************************************
Script to deploy a devoe DB schema in your Oracle environment.
Script to deploy a PyDin DB schema in your Oracle environment.
Just run the script then check if all objects created successfully.
*******************************************************************************/

create table de_schedule (
create table pd_schedule (
id number(*, 0),
job_name varchar2(30 char),
job_description varchar2(30 char),
Expand All @@ -28,23 +28,23 @@ create table de_schedule (
debug varchar2(1 char),
created date,
updated date,
constraint de_schedule_pk primary key (id)
constraint pd_schedule_pk primary key (id)
);

create sequence de_schedule_seq
create sequence pd_schedule_seq
increment by 1
start with 1
nocache;

create or replace trigger de_schedule_id_trg
before insert on de_schedule
create or replace trigger pd_schedule_id_trg
before insert on pd_schedule
for each row
begin
select de_schedule_seq.nextval into :new.id from dual;
select pd_schedule_seq.nextval into :new.id from dual;
end;
/

create table de_run_history (
create table pd_run_history (
id number(*, 0),
job_id number(*, 0),
run_mode varchar2(25 char),
Expand All @@ -68,32 +68,32 @@ create table de_run_history (
text_error clob,
data_dump blob,
updated date,
constraint de_run_history_pk primary key (id)
constraint pd_run_history_pk primary key (id)
);

create index de_run_history_job_ix
on de_run_history (job_id);
create index pd_run_history_job_ix
on pd_run_history (job_id);

create index de_run_history_start_date_ix
on de_run_history (start_date);
create index pd_run_history_start_date_ix
on pd_run_history (start_date);

create index de_run_history_status_ix
on de_run_history (status);
create index pd_run_history_status_ix
on pd_run_history (status);

create sequence de_run_history_seq
create sequence pd_run_history_seq
increment by 1
start with 1
nocache;

create or replace trigger de_run_history_id_trg
before insert on de_run_history
create or replace trigger pd_run_history_id_trg
before insert on pd_run_history
for each row
begin
select de_run_history_seq.nextval into :new.id from dual;
select pd_run_history_seq.nextval into :new.id from dual;
end;
/

create table de_task_history (
create table pd_task_history (
id number(*, 0),
job_id number(*, 0),
run_id number(*, 0),
Expand All @@ -108,23 +108,23 @@ create table de_task_history (
result_value number(*, 0),
result_long varchar2(4000 char),
updated date,
constraint de_task_history_pk primary key (id)
constraint pd_task_history_pk primary key (id)
);

create sequence de_task_history_seq
create sequence pd_task_history_seq
increment by 1
start with 1
nocache;

create or replace trigger de_task_history_id_trg
before insert on de_task_history
create or replace trigger pd_task_history_id_trg
before insert on pd_task_history
for each row
begin
select de_task_history_seq.nextval into :new.id from dual;
select pd_task_history_seq.nextval into :new.id from dual;
end;
/

create table de_step_history (
create table pd_step_history (
id number(*, 0),
job_id number(*, 0),
run_id number(*, 0),
Expand All @@ -144,23 +144,23 @@ create table de_step_history (
result_value number(*, 0),
result_long varchar2(4000 char),
updated date,
constraint de_step_history_pk primary key (id)
constraint pd_step_history_pk primary key (id)
);

create sequence de_step_history_seq
create sequence pd_step_history_seq
increment by 1
start with 1
nocache;

create or replace trigger de_step_history_id_trg
before insert on de_step_history
create or replace trigger pd_step_history_id_trg
before insert on pd_step_history
for each row
begin
select de_step_history_seq.nextval into :new.id from dual;
select pd_step_history_seq.nextval into :new.id from dual;
end;
/

create table de_file_log (
create table pd_file_log (
id number(*, 0),
job_id number(*, 0),
run_id number(*, 0),
Expand All @@ -172,23 +172,23 @@ create table de_file_log (
file_size number(*, 0),
start_date date,
end_date date,
constraint de_file_log_pk primary key (id)
constraint pd_file_log_pk primary key (id)
);

create sequence de_file_log_seq
create sequence pd_file_log_seq
increment by 1
start with 1
nocache;

create or replace trigger de_file_log_id_trg
before insert on de_file_log
create or replace trigger pd_file_log_id_trg
before insert on pd_file_log
for each row
begin
select de_file_log_seq.nextval into :new.id from dual;
select pd_file_log_seq.nextval into :new.id from dual;
end;
/

create table de_sql_log (
create table pd_sql_log (
id number(*, 0),
job_id number(*, 0),
run_id number(*, 0),
Expand All @@ -205,23 +205,23 @@ create table de_sql_log (
output_text varchar2(4000 char),
error_code number(*, 0),
error_text varchar2(4000 char),
constraint de_sql_log_pk primary key (id)
constraint pd_sql_log_pk primary key (id)
);

create sequence de_sql_log_seq
create sequence pd_sql_log_seq
increment by 1
start with 1
nocache;

create or replace trigger de_sql_log_id_trg
before insert on de_sql_log
create or replace trigger pd_sql_log_id_trg
before insert on pd_sql_log
for each row
begin
select de_sql_log_seq.nextval into :new.id from dual;
select pd_sql_log_seq.nextval into :new.id from dual;
end;
/

create table de_components (
create table pd_components (
id varchar2(100 char),
server varchar2(100 char),
user varchar2(100 char),
Expand All @@ -231,14 +231,14 @@ create table de_components (
start_date date,
stop_date date,
status varchar2(1 char),
constraint de_components_pk primary key (id)
constraint pd_components_pk primary key (id)
);

insert into de_components (id) values ('SCHEDULER');
insert into de_components (id) values ('RESTAPI');
insert into pd_components (id) values ('SCHEDULER');
insert into pd_components (id) values ('RESTAPI');
commit;

create table de_job_config (
create table pd_job_config (
job_id number(*, 0) not null,
node_seqno number(*, 0) not null,
node_name varchar2(100 char),
Expand Down
34 changes: 17 additions & 17 deletions db/sqlite.sql
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
/*******************************************************************************
Script to deploy a devoe DB schema in your SQLite environment.
Script to deploy a PyDin DB schema in your SQLite environment.
Just run the script then check if all objects created successfully.
*******************************************************************************/

create table de_schedule (
create table pd_schedule (
id integer primary key autoincrement,
job_name text,
job_description text,
Expand All @@ -30,7 +30,7 @@ create table de_schedule (
updated text
);

create table de_run_history (
create table pd_run_history (
id integer primary key autoincrement,
job_id integer,
run_mode text,
Expand All @@ -56,16 +56,16 @@ create table de_run_history (
updated text
);

create index de_run_history_job_ix
on de_run_history (job_id);
create index pd_run_history_job_ix
on pd_run_history (job_id);

create index de_run_history_start_date_ix
on de_run_history (start_date);
create index pd_run_history_start_date_ix
on pd_run_history (start_date);

create index de_run_history_status_ix
on de_run_history (status);
create index pd_run_history_status_ix
on pd_run_history (status);

create table de_task_history (
create table pd_task_history (
id integer primary key autoincrement,
job_id integer,
run_id integer,
Expand All @@ -82,7 +82,7 @@ create table de_task_history (
updated text
);

create table de_step_history (
create table pd_step_history (
id integer primary key autoincrement,
job_id integer,
run_id integer,
Expand All @@ -104,7 +104,7 @@ create table de_step_history (
updated text
);

create table de_file_log (
create table pd_file_log (
id integer primary key autoincrement,
job_id integer,
run_id integer,
Expand All @@ -118,7 +118,7 @@ create table de_file_log (
end_date text
);

create table de_sql_log (
create table pd_sql_log (
id integer primary key autoincrement,
job_id integer,
run_id integer,
Expand All @@ -137,7 +137,7 @@ create table de_sql_log (
error_text text
);

create table de_components (
create table pd_components (
id text primary key,
server text,
user text,
Expand All @@ -149,11 +149,11 @@ create table de_components (
status text
);

insert into de_components (id) values ('SCHEDULER');
insert into de_components (id) values ('RESTAPI');
insert into pd_components (id) values ('SCHEDULER');
insert into pd_components (id) values ('RESTAPI');
commit;

create table de_job_config (
create table pd_job_config (
job_id integer not null,
node_seqno integer not null,
node_name text,
Expand Down
6 changes: 0 additions & 6 deletions devoe/samples/scheduler.txt
View file

This file was deleted.

4 changes: 2 additions & 2 deletions devoe/__init__.py → pydin/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,13 @@
from .vars import run_id, task_id, step_id, process_id


__version__ = '0.0.2'
__version__ = '0.1.0'
__status__ = 'Development'

__author__ = 'Timur Faradzhov'
__email__ = 'timurfaradzhov@gmail.com'
__license__ = 'MIT'
__copyright__ = 'Copyright 2022, The Devoe Project'
__copyright__ = 'Copyright 2022, The PyDin Project'
__maintainer__ = __author__

__all__ = [Driver, Manager, Interface, Scheduler, Job, Pipeline, Mapper,
Expand Down
6 changes: 3 additions & 3 deletions devoe/api.py → pydin/api.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ def create_scheduler(self, name=None, desc=None, path=None):
logger.debug(f'File {dest} created')
else:
raise Exception(f'file {dest} already exists!')
config_path = os.path.join(root, 'devoe.ini')
config_path = os.path.join(root, 'pydin.ini')
config_local = configparser.ConfigParser()
config_dict = {'SCHEDULER': {'scheduler_name': name or '',
'scheduler_desc': desc or '',
Expand Down Expand Up @@ -207,7 +207,7 @@ def create_job(self, name=None, desc=None, mday=None, wday=None,
else:
raise Exception(f'file {dest} already exists!')

config_path = os.path.join(folder, 'devoe.ini')
config_path = os.path.join(folder, 'pydin.ini')
config_local = configparser.ConfigParser()
config_dict = {'JOB': {},
'LOGGING': {'console': 'True',
Expand Down Expand Up @@ -529,7 +529,7 @@ def push_repo(self, id=None, message=None):
count = len(staged_files)
logger.debug(f'{count} files staged')
if count > 0:
message = message or f'devoe[{timestamp}]'
message = message or f'pydin[{timestamp}]'
commit = repo.index.commit(message)
logger.debug(f'This {message} commit made as <{commit.hexsha}>')
origin = repo.remote('origin')
Expand Down
0 devoe/cache.py → pydin/cache.py
View file
File renamed without changes.
2 changes: 1 addition & 1 deletion devoe/cli.py → pydin/cli.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -428,7 +428,7 @@ def edit_config(self, type=None):
from .config import user_config as path
elif type.isdigit() is True:
id = self._check_id(type)
path = os.path.join(self.root, f'jobs/{id}/devoe.ini')
path = os.path.join(self.root, f'jobs/{id}/pydin.ini')
path = os.path.abspath(path)
if os.path.exists(path) is True:
# Launch edition process and wait until it is completed.
Expand Down
Loading

0 comments on commit 41a6c77

Please sign in to comment.