add ext_updated_at, fix notificator trouble

norbornen · Sep 19, 2019 · 375ace2 · 375ace2
1 parent 9e68b96
commit 375ace2
Show file tree

Hide file tree

Showing 8 changed files with 66 additions and 28 deletions.
diff --git a/src/factories/offer/flat/avito.ts b/src/factories/offer/flat/avito.ts
@@ -22,7 +22,6 @@ export default class AvitoExtEntityFactory extends AbstractExtEntityFactory {
         if (latitude && longitude) {
             location.coordinates = { latitude, longitude };
         }
-
         //
         const rooms_count = pathOr<number | string>('-', ['ext', 'rooms'], extFlatOffer);
         const floor_number = pathOr<number | string>('-', ['ext', 'floor'], extFlatOffer);
@@ -36,8 +35,15 @@ export default class AvitoExtEntityFactory extends AbstractExtEntityFactory {
             rooms_count: rooms_count && /^\d+$/.test(rooms_count + '') ? Number(rooms_count) : rooms_count,
             floor_number: floor_number && /^\d+$/.test(floor_number + '') ? Number(floor_number) : floor_number,
             floors_total: floors_total && /^\d+$/.test(floors_total + '') ? Number(floors_total) : floors_total,
-            price, location
+            price, location,
         };
+        if ('time' in extFlatOffer && /^\d+$/.test(`${extFlatOffer.time}`)) {
+            const ext_updated_at = new Date(Number(String(extFlatOffer.time).padEnd(13, '0')));
+            if (ext_updated_at && !isNaN(ext_updated_at.getTime())) {
+                offer.ext_updated_at = ext_updated_at;
+            }
+        }
+
         return new FlatOfferModel(offer);
     }
 }

diff --git a/src/factories/offer/flat/cian.ts b/src/factories/offer/flat/cian.ts
@@ -44,6 +44,17 @@ export default class CianExtEntityFactory extends AbstractExtEntityFactory {
             floors_total: path<number | string | null>(['building', 'floorsCount'], extFlatOffer)!,
             price, location
         };
+        let ext_updated_at: Date | undefined;
+        if ('creationDate' in extFlatOffer && !isNil(extFlatOffer.creationDate) && !isEmpty(extFlatOffer.creationDate)) {
+            ext_updated_at = new Date(extFlatOffer.creationDate);
+        }
+        if ((!ext_updated_at || isNaN(ext_updated_at.getTime())) && 'addedTimestamp' in extFlatOffer && !isNil(extFlatOffer.addedTimestamp) && /^\d+$/.test(`${extFlatOffer.addedTimestamp}`)) {
+            ext_updated_at = new Date(Number(String(extFlatOffer.addedTimestamp).padEnd(13, '0')));
+        }
+        if (ext_updated_at && !isNaN(ext_updated_at.getTime())) {
+            offer.ext_updated_at = ext_updated_at;
+        }
+
         return new FlatOfferModel(offer);
     }
 }

diff --git a/src/factories/offer/flat/realty_yandex.ts b/src/factories/offer/flat/realty_yandex.ts
@@ -22,7 +22,6 @@ export default class YandexExtEntityFactory extends AbstractExtEntityFactory {
         if (latitude && longitude) {
             location.coordinates = { latitude, longitude };
         }
-
         //
         const floor_number = ([] as number[]).concat(extFlatOffer.floorsOffered || [], extFlatOffer.floorsFlatOffered || []).find((x) => !isNil(x));
 
@@ -36,6 +35,17 @@ export default class YandexExtEntityFactory extends AbstractExtEntityFactory {
             floors_total: path<number | string | null>(['floorsTotal'], extFlatOffer)!,
             price, location
         };
+        let ext_updated_at: Date | undefined;
+        if ('updateDate' in extFlatOffer && !isNil(extFlatOffer.updateDate) && !isEmpty(extFlatOffer.updateDate)) {
+            ext_updated_at = new Date(extFlatOffer.updateDate);
+        }
+        if ((!ext_updated_at || isNaN(ext_updated_at.getTime())) && 'creationDate' in extFlatOffer && !isNil(extFlatOffer.creationDate) && !isEmpty(extFlatOffer.creationDate)) {
+            ext_updated_at = new Date(extFlatOffer.creationDate);
+        }
+        if (ext_updated_at && !isNaN(ext_updated_at.getTime())) {
+            offer.ext_updated_at = ext_updated_at;
+        }
+
         return new FlatOfferModel(offer);
     }
 }

diff --git a/src/models/ext_entity/ExtEntity.ts b/src/models/ext_entity/ExtEntity.ts
@@ -1,3 +1,4 @@
+import { isNil } from 'ramda';
 import { prop, Ref, instanceMethod, InstanceType } from '@hasezoey/typegoose';
 import { Model } from 'mongoose';
 import { CommonModel, CommonModelDTO } from '../CommonModel';
@@ -16,6 +17,9 @@ abstract class ExtEntity extends CommonModel {
     @prop({ select: false })
     public ext_data?: object;
 
+    @prop()
+    public ext_updated_at?: Date;
+
     @prop({ default: false })
     public is_notifications_send?: boolean;
 
@@ -25,14 +29,18 @@ abstract class ExtEntity extends CommonModel {
         const model: Model<InstanceType<ExtEntity>> = is_instanceof_model ? (this.constructor as never) : this.getModelForClass(this.constructor);
         const data = (this as InstanceType<this>).toObject();
         if (is_instanceof_model) {
-            delete data._id;
+            ['_id', '__v', 'is_notifications_send', 'is_active'].forEach((key) => delete data[key]);
         }
 
         const item = await model.findOneAndUpdate(
             { ext_id: data.ext_id, source: data.source },
             data,
             { new: true, upsert: true, setDefaultsOnInsert: true, runValidators: true }
         );
+        if (!('ext_updated_at' in item) || isNil(item.ext_updated_at)) {
+            item.ext_updated_at = new Date();
+            await item.save();
+        }
         Object.assign(this, item);
 
         return this;

diff --git a/src/models/ext_entity/offer/Offer.ts b/src/models/ext_entity/offer/Offer.ts
@@ -4,6 +4,7 @@ import { ExtEntity, ExtEntityDTO } from '../ExtEntity';
 
 @index({ ext_id: 1, source: 1 }, { unique: true })
 @index({ __t: 1, is_active: 1, createdAt: 1 })
+@index({ __t: 1, is_active: 1, ext_updated_at: 1 })
 @index({ is_active: 1 })
 abstract class Offer extends ExtEntity {
 

diff --git a/src/scripts/notifier.ts b/src/scripts/notifier.ts
@@ -5,12 +5,12 @@ import { NotificationUsecase } from '../usecases/NotificationUsecase';
 enum ExtCode {'OK', 'FAIL'}
 
 (async () => {
+    console.info('[notifier::start] ', new Date().toString());
     let extcode: ExtCode = ExtCode.OK;
     let connection;
     try {
         connection = await createConnection({db: 'mongodb://localhost/cyanic'});
 
-        console.info('[notifier] ', new Date().toString());
         const usecase = new NotificationUsecase();
         await usecase.sendNotifications();
     } catch (err) {
@@ -21,5 +21,6 @@ enum ExtCode {'OK', 'FAIL'}
     if (connection) {
         await disconnect();
     }
+    console.info('[notifier::done] ', new Date().toString(), '\n');
     process.exit(extcode);
 })();
diff --git a/src/scripts/updater.ts b/src/scripts/updater.ts
@@ -5,12 +5,12 @@ import { ImportUsecase } from '../usecases/ImportUsecase';
 enum ExtCode {'OK', 'FAIL'}
 
 (async () => {
+    console.info('[updater::start] ', new Date().toString());
     let extcode: ExtCode = ExtCode.OK;
     let connection;
     try {
         connection = await createConnection({db: 'mongodb://localhost/cyanic'});
 
-        console.info('[updater] ', new Date().toString());
         const usecase = new ImportUsecase();
         await usecase.getAndUpdateExtEntities();
     } catch (err) {
@@ -21,5 +21,6 @@ enum ExtCode {'OK', 'FAIL'}
     if (connection) {
         await disconnect();
     }
+    console.info('[updater::done] ', new Date().toString(), '\n');
     process.exit(extcode);
 })();
diff --git a/yarn.lock b/yarn.lock
@@ -18,7 +18,7 @@
     esutils "^2.0.2"
     js-tokens "^4.0.0"
 
-"@hasezoey/typegoose@^5.9.0":
+"@hasezoey/typegoose@^5.9.2":
   version "5.9.2"
   resolved "https://registry.yarnpkg.com/@hasezoey/typegoose/-/typegoose-5.9.2.tgz#10e949ba04da61156416448f9d5990794dd15fd3"
   integrity sha512-PwbOGJi9IggUF8E2K3fEHSEoBgXV3nlDd1H6m/rjC9iWG1GeKHAp9gp0SPZh3e/5rJ3vWbfWbyukD/pMT3tbxA==
@@ -73,12 +73,12 @@
   resolved "https://registry.yarnpkg.com/@types/qs/-/qs-6.5.3.tgz#1c3b71b091eaeaf5924538006b7f70603ce63d38"
   integrity sha512-Jugo5V/1bS0fRhy2z8+cUAHEyWOATaz4rbyLVvcFs7+dXp5HfwpEwzF1Q11bB10ApUqHf+yTauxI0UXQDwGrbA==
 
-"@types/ramda@^0.26.21":
-  version "0.26.21"
-  resolved "https://registry.yarnpkg.com/@types/ramda/-/ramda-0.26.21.tgz#cfd552c679453d153a9e2fd89ffc5dd5aa93d2c9"
-  integrity sha512-zMYtIZMceA6BvH+or6LmewLBgojbXg5+FGCwjO8K+Z+d/ZWxILmhhASXkehW0PqJL+V0QbyDeeAHix0dvEKXfQ==
+"@types/ramda@^0.26.22":
+  version "0.26.22"
+  resolved "https://registry.yarnpkg.com/@types/ramda/-/ramda-0.26.22.tgz#4506cf76b53191d17fce473bc3de529ece6f4886"
+  integrity sha512-TfX4BwgjMSXFuyb1QZsY2D8KEsJfiWolzJHYzFWyB/0UGiyX6BFvWX05IzzrMAsZpB4uN0GXAL5QcJHTKnBxoQ==
   dependencies:
-    ts-toolbelt "^3.8.4"
+    ts-toolbelt "^3.8.75"
 
 agent-base@4, agent-base@^4.2.0, agent-base@^4.3.0:
   version "4.3.0"
@@ -521,10 +521,10 @@ mongoose-legacy-pluralize@1.0.2:
   resolved "https://registry.yarnpkg.com/mongoose-legacy-pluralize/-/mongoose-legacy-pluralize-1.0.2.tgz#3ba9f91fa507b5186d399fb40854bff18fb563e4"
   integrity sha512-Yo/7qQU4/EyIS8YDFSeenIvXxZN+ld7YdV9LqFVQJzTLye8unujAWPZ4NWKfFA+RNjh+wvTWKY9Z3E5XM6ZZiQ==
 
-mongoose@^5.7.0:
-  version "5.7.0"
-  resolved "https://registry.yarnpkg.com/mongoose/-/mongoose-5.7.0.tgz#5a686d7a85342784a26d1afb875a2a8170a7a32f"
-  integrity sha512-nE43r4kEUpUBfr32rf+zJKEtzd6I3F5YdbrSHp/DDdVQyej34Cv7gfBdcoRNehrPQDV3khOh0JpiS1aLN9/OCw==
+mongoose@^5.7.1:
+  version "5.7.1"
+  resolved "https://registry.yarnpkg.com/mongoose/-/mongoose-5.7.1.tgz#47f8feb3d28e9caf645859e4245a933b39c1797b"
+  integrity sha512-TJeZHb5h8UGH++ctngh2shgZuZmR9o0D8elxfkWzIpKB6QAFsJPmALtDuD6RYZTS33+CifhDdAMGqs3OpJ3cyQ==
   dependencies:
     async "2.6.2"
     bson "~1.1.1"
@@ -888,21 +888,21 @@ ts-money@^0.4.6:
   dependencies:
     lodash "^4.17.5"
 
-ts-node@^8.3.0:
-  version "8.3.0"
-  resolved "https://registry.yarnpkg.com/ts-node/-/ts-node-8.3.0.tgz#e4059618411371924a1fb5f3b125915f324efb57"
-  integrity sha512-dyNS/RqyVTDcmNM4NIBAeDMpsAdaQ+ojdf0GOLqE6nwJOgzEkdRNzJywhDfwnuvB10oa6NLVG1rUJQCpRN7qoQ==
+ts-node@^8.4.1:
+  version "8.4.1"
+  resolved "https://registry.yarnpkg.com/ts-node/-/ts-node-8.4.1.tgz#270b0dba16e8723c9fa4f9b4775d3810fd994b4f"
+  integrity sha512-5LpRN+mTiCs7lI5EtbXmF/HfMeCjzt7DH9CZwtkr6SywStrNQC723wG+aOWFiLNn7zT3kD/RnFqi3ZUfr4l5Qw==
   dependencies:
     arg "^4.1.0"
     diff "^4.0.1"
     make-error "^1.1.1"
     source-map-support "^0.5.6"
     yn "^3.0.0"
 
-ts-toolbelt@^3.8.4:
-  version "3.8.63"
-  resolved "https://registry.yarnpkg.com/ts-toolbelt/-/ts-toolbelt-3.8.63.tgz#ee5a94a5d870f437bc6811f7428ea24c78821f6a"
-  integrity sha512-OlsFC8uF1mpN2O7DlvSkgnXzOvubO790eFhTvZBFI+WLge2BYzkQk73Q3yF0R8usxhkqbemLomqRhCGWasoIKw==
+ts-toolbelt@^3.8.75:
+  version "3.14.0"
+  resolved "https://registry.yarnpkg.com/ts-toolbelt/-/ts-toolbelt-3.14.0.tgz#f4c4984cf4e988915a0175fd6a8ded00bd1d23b0"
+  integrity sha512-DYjD8tL7M1kBogRd9UKg3bUP5yh69WWcMSaA3By8ATiJU9fgYudSYIe8tWD5cpPkrGCdGGnKXQHG+5IrjJ5uhQ==
 
 tslib@^1.8.0, tslib@^1.8.1:
   version "1.10.0"
@@ -942,10 +942,10 @@ type-check@~0.3.2:
   dependencies:
     prelude-ls "~1.1.2"
 
-typescript@^3.6.2:
-  version "3.6.2"
-  resolved "https://registry.yarnpkg.com/typescript/-/typescript-3.6.2.tgz#105b0f1934119dde543ac8eb71af3a91009efe54"
-  integrity sha512-lmQ4L+J6mnu3xweP8+rOrUwzmN+MRAj7TgtJtDaXE5PMyX2kCrklhg3rvOsOIfNeAWMQWO2F1GPc1kMD2vLAfw==
+typescript@^3.6.3:
+  version "3.6.3"
+  resolved "https://registry.yarnpkg.com/typescript/-/typescript-3.6.3.tgz#fea942fabb20f7e1ca7164ff626f1a9f3f70b4da"
+  integrity sha512-N7bceJL1CtRQ2RiG0AQME13ksR7DiuQh/QehubYcghzv20tnh+MQnQIuJddTmsbqYj+dztchykemz0zFzlvdQw==
 
 unpipe@1.0.0:
   version "1.0.0"