feat: migrate logs data from browser.storage to indexeddb

package.json

@@ -42,12 +42,14 @@
     "@tiptap/starter-kit": "^2.0.0-beta.181",
     "@tiptap/vue-3": "^2.0.0-beta.90",
     "@viselect/vanilla": "^3.0.0-beta.13",
+    "@vueuse/rxjs": "^8.6.0",
     "@vuex-orm/core": "^0.36.4",
     "compare-versions": "^4.1.2",
     "crypto-js": "^4.1.1",
     "css-selector-generator": "^3.6.0",
     "dayjs": "^1.10.7",
     "defu": "^6.0.0",
+    "dexie": "^3.2.2",
     "drawflow": "^0.0.58",
     "idb": "^7.0.0",
     "lodash.clonedeep": "^4.5.0",
@@ -56,9 +58,11 @@
     "nanoid": "^3.2.0",
     "object-path": "^0.11.8",
     "papaparse": "^5.3.1",
+    "pinia": "^2.0.14",
+    "rxjs": "^7.5.5",
     "tippy.js": "^6.3.1",
     "v-remixicon": "^0.1.1",
-    "vue": "^3.2.31",
+    "vue": "^3.2.37",
     "vue-i18n": "^9.2.0-beta.29",
     "vue-router": "^4.0.11",
     "vue-toastification": "^2.0.0-rc.5",
@@ -72,7 +76,7 @@
     "@babel/preset-env": "7.15.6",
     "@intlify/vue-i18n-loader": "^4.2.0",
     "@tailwindcss/typography": "^0.5.1",
-    "@vue/compiler-sfc": "3.2.19",
+    "@vue/compiler-sfc": "^3.2.37",
     "archiver": "^5.3.0",
     "autoprefixer": "10.3.6",
     "babel-loader": "^8.2.2",
@@ -102,7 +106,7 @@
     "source-map-loader": "3.0.0",
     "tailwindcss": "^3.0.7",
     "terser-webpack-plugin": "5.2.4",
-    "vue-loader": "16.8.1",
+    "vue-loader": "^17.0.0",
     "webpack": "5.55.1",
     "webpack-cli": "4.9.2",
     "webpack-dev-server": "3.11.2"

src/background/WorkflowLogger.js

@@ -1,15 +1,19 @@
+import dbLogs, { defaultLogItem } from '@/db/logs';
+/* eslint-disable class-methods-use-this */
 class WorkflowLogger {
-  constructor({ storage, key = 'logs' }) {
+  constructor({ key = 'logs' }) {
     this.key = key;
-    this.storage = storage;
   }
 
-  async add(data) {
-    const logs = (await this.storage.get(this.key)) || [];
+  async add({ detail, history, ctxData, data }) {
+    const logDetail = { ...defaultLogItem, ...detail };
 
-    logs.unshift(data);
-
-    await this.storage.set(this.key, logs);
+    await Promise.all([
+      dbLogs.logsData.add(data),
+      dbLogs.ctxData.add(ctxData),
+      dbLogs.items.add(logDetail),
+      dbLogs.histories.add(history),
+    ]);
   }
 }
 

src/background/workflowEngine/engine.js

@@ -17,6 +17,7 @@ class WorkflowEngine {
     this.parentWorkflow = parentWorkflow;
     this.saveLog = workflow.settings?.saveLog ?? true;
 
+    this.workerId = 0;
     this.workers = new Map();
     this.waitConnections = {};
 
@@ -166,7 +167,10 @@ class WorkflowEngine {
   }
 
   addWorker(detail) {
-    const worker = new Worker(this);
+    this.workerId += 1;
+
+    const workerId = `worker-${this.workerId}`;
+    const worker = new Worker(workerId, this);
     worker.init(detail);
 
     this.workers.set(worker.id, worker);
@@ -178,7 +182,7 @@ class WorkflowEngine {
     const isLimit = this.history.length >= 1001;
     const notErrorLog = detail.type !== 'error';
 
-    if ((!this.saveLog || isLimit) && notErrorLog) return;
+    if (!this.saveLog && isLimit && notErrorLog) return;
 
     this.logHistoryId += 1;
     detail.id = this.logHistoryId;
@@ -273,24 +277,31 @@ class WorkflowEngine {
       if (!this.workflow.isTesting) {
         const { name, id } = this.workflow;
 
-        let { logsCtxData } = await browser.storage.local.get('logsCtxData');
-        if (!logsCtxData) logsCtxData = {};
-        logsCtxData[this.id] = this.historyCtxData;
-        await browser.storage.local.set({ logsCtxData });
-
         await this.logger.add({
-          name,
-          status,
-          message,
-          id: this.id,
-          workflowId: id,
-          endedAt: endedTimestamp,
-          parentLog: this.parentWorkflow,
-          startedAt: this.startedTimestamp,
-          history: this.saveLog ? this.history : [],
+          detail: {
+            name,
+            status,
+            message,
+            id: this.id,
+            workflowId: id,
+            endedAt: endedTimestamp,
+            parentLog: this.parentWorkflow,
+            startedAt: this.startedTimestamp,
+          },
+          history: {
+            logId: this.id,
+            data: this.saveLog ? this.history : [],
+          },
+          ctxData: {
+            logId: this.id,
+            data: this.historyCtxData,
+          },
           data: {
-            table: this.referenceData.table,
-            variables: this.referenceData.variables,
+            logId: this.id,
+            data: {
+              table: this.referenceData.table,
+              variables: this.referenceData.variables,
+            },
           },
         });
       }

src/background/workflowEngine/worker.js

@@ -1,13 +1,12 @@
-import { nanoid } from 'nanoid';
 import browser from 'webextension-polyfill';
 import { toCamelCase, sleep, objectHasKey, isObject } from '@/utils/helper';
 import { tasks } from '@/utils/shared';
 import referenceData from '@/utils/referenceData';
 import { convertData, waitTabLoaded, getBlockConnection } from './helper';
 
 class Worker {
-  constructor(engine) {
-    this.id = nanoid(5);
+  constructor(id, engine) {
+    this.id = id;
     this.engine = engine;
     this.settings = engine.workflow.settings;
 

src/components/newtab/logs/LogsDataViewer.vue

@@ -1,50 +1,61 @@
 <template>
-  <div class="flex items-center mb-2">
-    <ui-input
-      v-model="state.fileName"
-      :placeholder="t('common.fileName')"
-      :title="t('common.fileName')"
-    />
-    <div class="flex-grow"></div>
-    <ui-popover trigger-width>
-      <template #trigger>
-        <ui-button variant="accent">
-          <span>{{ t('log.exportData.title') }}</span>
-          <v-remixicon name="riArrowDropDownLine" class="ml-2 -mr-1" />
-        </ui-button>
-      </template>
-      <ui-list class="space-y-1">
-        <ui-list-item
-          v-for="type in dataExportTypes"
-          :key="type.id"
-          v-close-popover
-          class="cursor-pointer"
-          @click="exportData(type.id)"
-        >
-          {{ t(`log.exportData.types.${type.id}`) }}
-        </ui-list-item>
-      </ui-list>
-    </ui-popover>
+  <div v-if="state.status === 'loading'" class="text-center py-8">
+    <ui-spinner color="text-primary" />
   </div>
-  <ui-tabs v-if="objectHasKey(log.data, 'table')" v-model="state.activeTab">
-    <ui-tab value="table">
-      {{ t('workflow.table.title') }}
-    </ui-tab>
-    <ui-tab value="variables">
-      {{ t('workflow.variables.title', 2) }}
-    </ui-tab>
-  </ui-tabs>
-  <shared-codemirror
-    :model-value="dataStr"
-    :class="editorClass"
-    class="rounded-t-none"
-    lang="json"
-    readonly
-  />
+  <template v-else-if="state.status === 'idle'">
+    <div class="flex items-center mb-2">
+      <ui-input
+        v-model="state.fileName"
+        :placeholder="t('common.fileName')"
+        :title="t('common.fileName')"
+      />
+      <div class="flex-grow"></div>
+      <ui-popover trigger-width>
+        <template #trigger>
+          <ui-button variant="accent">
+            <span>{{ t('log.exportData.title') }}</span>
+            <v-remixicon name="riArrowDropDownLine" class="ml-2 -mr-1" />
+          </ui-button>
+        </template>
+        <ui-list class="space-y-1">
+          <ui-list-item
+            v-for="type in dataExportTypes"
+            :key="type.id"
+            v-close-popover
+            class="cursor-pointer"
+            @click="exportData(type.id)"
+          >
+            {{ t(`log.exportData.types.${type.id}`) }}
+          </ui-list-item>
+        </ui-list>
+      </ui-popover>
+    </div>
+    <ui-tabs v-if="objectHasKey(logsData, 'table')" v-model="state.activeTab">
+      <ui-tab value="table">
+        {{ t('workflow.table.title') }}
+      </ui-tab>
+      <ui-tab value="variables">
+        {{ t('workflow.variables.title', 2) }}
+      </ui-tab>
+    </ui-tabs>
+    <shared-codemirror
+      :model-value="dataStr"
+      :class="editorClass"
+      class="rounded-t-none"
+      lang="json"
+      readonly
+    />
+  </template>
 </template>
 <script setup>
-import { shallowReactive, computed, defineAsyncComponent } from 'vue';
+import {
+  shallowReactive,
+  computed,
+  defineAsyncComponent,
+  onMounted,
+} from 'vue';
 import { useI18n } from 'vue-i18n';
+import dbLogs from '@/db/logs';
 import { dataExportTypes } from '@/utils/shared';
 import { objectHasKey } from '@/utils/helper';
 import dataExporter from '@/utils/dataExporter';
@@ -67,35 +78,40 @@ const props = defineProps({
 const { t } = useI18n();
 
 const state = shallowReactive({
+  status: 'loading',
   activeTab: 'table',
   fileName: props.log.name,
 });
-const cache = {
+const logsData = {
   table: '',
   variables: '',
 };
 
 const dataStr = computed(() => {
-  if (cache[state.activeTab]) return cache[state.activeTab];
-
-  let { data } = props.log;
-
-  if (objectHasKey(props.log.data, 'table')) {
-    data = props.log.data[state.activeTab];
-  }
-
-  data = JSON.stringify(data, null, 2);
-  /* eslint-disable-next-line */
-  cache[state.activeTab] = data;
+  if (state.status !== 'idle') return '';
 
-  return data;
+  return logsData[state.activeTab] ? logsData[state.activeTab] : '';
 });
 
 function exportData(type) {
   dataExporter(
-    props.log.data?.table || props.log.data,
+    logsData?.table || logsData,
     { name: state.fileName, type },
     true
   );
 }
+
+onMounted(async () => {
+  const data = await dbLogs.logsData.where('logId').equals(props.log.id).last();
+
+  if (!data) {
+    state.status = 'error';
+    return;
+  }
+
+  Object.keys(data.data).forEach((key) => {
+    logsData[key] = JSON.stringify(data.data[key], null, 2);
+  });
+  state.status = 'idle';
+});
 </script>

src/components/newtab/shared/SharedLogsTable.vue

@@ -65,18 +65,14 @@ function formatDate(date, format) {
 
   return dayjs(date).format(format);
 }
-function getErrorMessage({ history, message }) {
+function getErrorMessage({ message }) {
   const messagePath = `log.messages.${message}`;
 
   if (message && te(messagePath)) {
     return t(messagePath);
   }
 
-  const lastHistory = history[history.length - 1];
-
-  return lastHistory && lastHistory.type === 'error'
-    ? lastHistory.message
-    : null;
+  return '';
 }
 </script>
 <style scoped>

src/composable/liveQuery.js

@@ -0,0 +1,6 @@
+import { liveQuery } from 'dexie';
+import { useObservable } from '@vueuse/rxjs';
+
+export function useLiveQuery(querier) {
+  return useObservable(liveQuery(querier));
+}

src/db/logs.js

@@ -0,0 +1,22 @@
+import Dexie from 'dexie';
+
+const dbLogs = new Dexie('logs');
+dbLogs.version(1).stores({
+  ctxData: '++id, logId',
+  logsData: '++id, logId',
+  histories: '++id, logId',
+  items: '++id, name, endedAt, workflowId, status, collectionId',
+});
+
+export const defaultLogItem = {
+  name: '',
+  endedAt: 0,
+  message: '',
+  startedAt: 0,
+  parentLog: null,
+  workflowId: null,
+  status: 'success',
+  collectionId: null,
+};
+
+export default dbLogs;