Rework FileManager Model

db/oracle.sql

@@ -130,8 +130,8 @@ create table pd_task_history (
   records_error     number(*, 0),
   files_read        number(*, 0),
   files_written     number(*, 0),
-  volume_read       number(*, 0),
-  volume_written    number(*, 0),
+  bytes_read        number(*, 0),
+  bytes_written     number(*, 0),
   result_value      number(*, 0),
   result_long       varchar2(4000 char),
   updated           date,
@@ -171,8 +171,8 @@ create table pd_step_history (
   records_error     number(*, 0),
   files_read        number(*, 0),
   files_written     number(*, 0),
-  volume_read       number(*, 0),
-  volume_written    number(*, 0),
+  bytes_read        number(*, 0),
+  bytes_written     number(*, 0),
   result_value      number(*, 0),
   result_long       varchar2(4000 char),
   updated           date,
@@ -193,17 +193,17 @@ end;
 /
 
 create table pd_file_log (
-  id         number(*, 0),
-  job_id     number(*, 0),
-  run_id     number(*, 0),
-  task_id    number(*, 0),
-  step_id    number(*, 0),
-  server     varchar2(50 char),
-  file_name  varchar2(200 char),
-  file_date  date,
-  file_size  number(*, 0),
-  start_date date,
-  end_date   date,
+  id          number(*, 0),
+  job_id      number(*, 0),
+  run_id      number(*, 0),
+  task_id     number(*, 0),
+  step_id     number(*, 0),
+  server_name varchar2(50 char),
+  file_name   varchar2(200 char),
+  file_date   date,
+  file_size   number(*, 0),
+  start_date  date,
+  end_date    date,
   constraint pd_file_log_pk primary key (id)
 );
 

db/sqlite.sql

@@ -100,8 +100,8 @@ create table pd_task_history (
   records_error     integer,
   files_read        integer,
   files_written     integer,
-  volume_read       integer,
-  volume_written    integer,
+  bytes_read        integer,
+  bytes_written     integer,
   result_value      integer,
   result_long       text,
   updated           text
@@ -127,8 +127,8 @@ create table pd_step_history (
   records_error     integer,
   files_read        integer,
   files_written     integer,
-  volume_read       integer,
-  volume_written    integer,
+  bytes_read        integer,
+  bytes_written     integer,
   result_value      integer,
   result_long       text,
   updated           text
@@ -140,7 +140,7 @@ create table pd_file_log (
   run_id      integer,
   task_id     integer,
   step_id     integer,
-  server      text,
+  server_name text,
   file_name   text,
   file_date   text,
   file_size   integer,

pydin/__init__.py

@@ -8,7 +8,7 @@
 from .models import Mapper
 from .models import Table, SQL, Select, Insert
 from .models import CSV, JSON, XML
-from .models import Files, FileManager
+from .models import Filenames, FileManager
 
 from .fields import run_id, task_id, step_id, process_id
 
@@ -23,5 +23,5 @@
 __maintainer__ = __author__
 
 __all__ = [Driver, Manager, Server, Scheduler, Job, Pipeline, Mapper,
-           Table, SQL, Select, Insert, CSV, JSON, XML, Files, FileManager,
+           Table, SQL, Select, Insert, CSV, JSON, XML, Filenames, FileManager,
            run_id, task_id, step_id, process_id]

pydin/core.py

@@ -2093,6 +2093,10 @@ def __init__(self, task_name=None, pipeline=None):
         self._records_written = 0
         self._records_processed = 0
         self._records_error = 0
+        self._files_read = 0
+        self._files_written = 0
+        self._bytes_read = 0
+        self._bytes_written = 0
         self._result_value = 0
         self._result_long = 0
         self.errors = set()
@@ -2205,6 +2209,50 @@ def records_error(self, value):
             self.logger.table(records_error=self.records_error)
         pass
 
+    @property
+    def files_read(self):
+        """Get number of files read."""
+        return self._files_read
+
+    @files_read.setter
+    def files_read(self, value):
+        if isinstance(value, int):
+            self._files_read += value
+            self.logger.table(files_read=self.files_read)
+
+    @property
+    def files_written(self):
+        """Get number of files written."""
+        return self._files_written
+
+    @files_written.setter
+    def files_written(self, value):
+        if isinstance(value, int):
+            self._files_written += value
+            self.logger.table(files_written=self.files_written)
+
+    @property
+    def bytes_read(self):
+        """Get number of bytes read."""
+        return self._bytes_read
+
+    @bytes_read.setter
+    def bytes_read(self, value):
+        if isinstance(value, int):
+            self._bytes_read += value
+            self.logger.table(bytes_read=self.bytes_read)
+
+    @property
+    def bytes_written(self):
+        """Get number of bytes written."""
+        return self._bytes_written
+
+    @bytes_written.setter
+    def bytes_written(self, value):
+        if isinstance(value, int):
+            self._bytes_written += value
+            self.logger.table(bytes_written=self.bytes_written)
+
     @property
     def result_value(self):
         """Get short numeric result value."""
@@ -2413,6 +2461,10 @@ def __init__(self, step_name=None, a=None, b=None, c=None, pipeline=None):
         self._records_written = 0
         self._records_processed = 0
         self._records_error = 0
+        self._files_read = 0
+        self._files_written = 0
+        self._bytes_read = 0
+        self._bytes_written = 0
         self._result_value = 0
         self._result_long = 0
         self.errors = set()
@@ -2576,13 +2628,60 @@ def records_error(self):
 
     @records_error.setter
     def records_error(self, value):
-        print('Hello there!')
         if isinstance(value, int):
             self._records_error += value
             self.task.records_error = value
             self.logger.table(records_error=self.records_error)
         pass
 
+    @property
+    def files_read(self):
+        """Get number of files read."""
+        return self._files_read
+
+    @files_read.setter
+    def files_read(self, value):
+        if isinstance(value, int):
+            self._files_read += value
+            self.task.files_read = value
+            self.logger.table(files_read=self.files_read)
+
+    @property
+    def files_written(self):
+        """Get number of files written."""
+        return self._files_written
+
+    @files_written.setter
+    def files_written(self, value):
+        if isinstance(value, int):
+            self._files_written += value
+            self.task.files_written = value
+            self.logger.table(files_written=self.files_written)
+
+    @property
+    def bytes_read(self):
+        """Get number of bytes read."""
+        return self._bytes_read
+
+    @bytes_read.setter
+    def bytes_read(self, value):
+        if isinstance(value, int):
+            self._bytes_read += value
+            self.task.bytes_read = value
+            self.logger.table(bytes_read=self.bytes_read)
+
+    @property
+    def bytes_written(self):
+        """Get number of bytes written."""
+        return self._bytes_written
+
+    @bytes_written.setter
+    def bytes_written(self, value):
+        if isinstance(value, int):
+            self._bytes_written += value
+            self.task.bytes_written = value
+            self.logger.table(bytes_written=self.bytes_written)
+
     @property
     def result_value(self):
         """Get short numeric result value."""