add mae

config.py

@@ -216,10 +216,12 @@ def create_parser():
                        help='Num of cycles for cosine decay and cyclic (default=1)')
     group.add_argument('--cycle_decay', type=float, default=1.0,
                        help='Decay rate of lr max in each cosine cycle (default=1.0)')
+    group.add_argument('--layer_decay', type=float, default=None,
+                       help='layer(model) decay rate of lr (default=None)')
 
     # Loss parameters
     group = parser.add_argument_group('Loss parameters')
-    group.add_argument('--loss', type=str, default='CE', choices=['BCE', 'CE'],
+    group.add_argument('--loss', type=str, default='CE', choices=['BCE', 'CE', 'None'],
                        help='Type of loss, BCE (BinaryCrossEntropy) or CE (CrossEntropy)  (default="CE")')
     group.add_argument('--label_smoothing', type=float, default=0.0,
                        help='Use label smoothing (default=0.0)')
@@ -270,6 +272,25 @@ def create_parser():
     group.add_argument('--train_url', type=str, default='/cache/output/',
                        help='model folder to save/load')
 
+    # pre-train
+    group = parser.add_argument_group('pre-train')
+    group.add_argument('--pretrain_resize', type=list, default=[224],
+                       help='Crop the size of the image for pre-training.'
+                            'The length of list should be 2 if tokenizer is required. (default=[224])')
+    group.add_argument('--pretrain_interpolations', type=list, default=['bicubic', 'bilinear'],
+                       help='Image interpolation mode for resize operator for pre-trainin')
+    group.add_argument('--tokenizer', type=str, default=None,
+                       help='Name of tokenizer model for pre-train')
+    group.add_argument('--tokenizer_ckpt_path', type=str, default='',
+                       help='Initialize tokenizer model from this checkpoint')
+    group.add_argument('--mask_type', type=str, default='random',
+                       choices=['block_wise', 'patch_aligned', 'random'],
+                       help='Type of mask generator')
+    group.add_argument('--mask_ratio', type=float, default=0.75,
+                       help='Masking ratio')
+    group.add_argument('--mask_patch_size', type=int, default=32,
+                       help='Size of mask patch')
+
     return parser_config, parser
 # fmt: on
 

configs/mae/mae_b_16_224_finetune_ascend.yaml

@@ -0,0 +1,58 @@
+# system
+mode: 0
+distribute: True
+num_parallel_workers: 8
+val_while_train: True
+
+# dataset
+dataset: "imagenet"
+data_dir: "/path/to/imagenet"
+shuffle: True
+dataset_download: False
+batch_size: 32
+drop_remainder: True
+
+# augmentation
+image_resize: 224
+scale: [0.08, 1.0]
+ratio: [0.75, 1.333]
+hflip: 0.5
+interpolation: "bicubic"
+auto_augment: "randaug-m9-mstd0.5-inc1"
+re_prob: 0.25
+mixup: 0.8
+cutmix: 1.0
+re_value: "random"
+
+# model
+model: "mae_b_16_224_finetune"
+drop_rate: 0.0
+drop_path_rate: 0.1
+pretrained: False
+ckpt_path: ""
+keep_checkpoint_max: 10
+ckpt_save_dir: "./ckpt"
+epoch_size: 100
+dataset_sink_mode: True
+amp_level: "O2"
+
+# loss
+loss: "CE"
+loss_scale: 1024.0
+label_smoothing: 0.1
+
+# lr scheduler
+scheduler: "warmup_cosine_decay"
+lr: 5e-4
+min_lr: 1e-6
+warmup_epochs: 5
+warmup_factor: 0
+decay_epochs: 95
+layer_decay: 0.65
+lr_epoch_stair: False
+
+# optimizer
+opt: "adamw"
+weight_decay: 0.05
+filter_bias_and_bn: True
+use_nesterov: False

configs/mae/mae_b_16_224_pretrain_ascend.yaml

@@ -0,0 +1,57 @@
+# system
+mode: 0
+distribute: True
+num_parallel_workers: 8
+
+# dataset
+dataset: "imagenet"
+data_dir: "/path/to/imagenet"
+shuffle: True
+dataset_download: False
+batch_size: 64
+drop_remainder: True
+
+# augmentation
+scale: [0.2, 1.0]
+ratio: [0.75, 1.333]
+hflip: 0.5
+color_jitter: [0.4, 0.4, 0.4]
+
+# model
+model: "mae_b_16_224_pretrain"
+drop_rate: 0.0
+drop_path_rate: 0.0
+pretrained: False
+ckpt_path: ""
+keep_checkpoint_max: 10
+ckpt_save_dir: "./ckpt"
+epoch_size: 800
+dataset_sink_mode: True
+amp_level: "O2"
+clip_grad: True
+clip_value: 3.0
+
+# loss
+loss: "None"
+loss_scale: 1024.0
+
+# lr scheduler
+scheduler: "warmup_cosine_decay"
+lr: 1.5e-4
+min_lr: 0
+warmup_epochs: 40
+warmup_factor: 0
+decay_epochs: 760
+lr_epoch_stair: False
+
+# optimizer
+opt: "adamw"
+weight_decay: 0.05
+filter_bias_and_bn: True
+use_nesterov: False
+
+# pre-train
+pretrain_resize: [224]
+pretrain_interpolations: ["bicubic"]
+mask_type: "random"
+mask_ratio: 0.75