First model version

configs/pretrain_language_model.yaml

@@ -0,0 +1,45 @@
+global:
+  name: pretrain-language-model
+  phase: train
+  stage: pretrain-language
+  workdir: workdir
+  seed: ~
+ 
+dataset:
+  train: {
+    roots: ['data/WikiText-103.csv'],
+    batch_size: 4096
+  }
+  test: {
+    roots: ['data/WikiText-103_eval_d1.csv'],
+    batch_size: 4096
+  }
+
+training:
+  epochs: 80
+  show_iters: 50
+  eval_iters: 6000
+  save_iters: 3000
+
+optimizer:
+  type: Adam
+  true_wd: False
+  wd: 0.0
+  bn_wd: False
+  clip_grad: 20
+  lr: 0.0001
+  args: {
+    betas: !!python/tuple [0.9, 0.999], # for default Adam 
+  }
+  scheduler: {
+    periods: [70, 10],
+    gamma: 0.1,
+  }
+
+model:
+  name: 'modules.model_language.BCNLanguage'
+  language: {
+    num_layers: 4,
+    loss_weight: 1.,
+    use_self_attn: False
+  }

configs/pretrain_vision_model.yaml

@@ -0,0 +1,58 @@
+global:
+  name: pretrain-vision-model
+  phase: train
+  stage: pretrain-vision
+  workdir: workdir
+  seed: ~
+ 
+dataset:
+  train: {
+    roots: ['data/training/MJ/MJ_train/', 
+            'data/training/MJ/MJ_test/', 
+            'data/training/MJ/MJ_valid/', 
+            'data/training/ST'],
+    batch_size: 384
+  }
+  test: {
+    roots: ['data/evaluation/IIIT5k_3000', 
+            'data/evaluation/SVT', 
+            'data/evaluation/SVTP',
+            'data/evaluation/IC13_857',
+            'data/evaluation/IC15_1811',
+            'data/evaluation/CUTE80'],
+    batch_size: 384
+  }
+  data_aug: True
+  multiscales: False
+  num_workers: 14
+
+training:
+  epochs: 8
+  show_iters: 50
+  eval_iters: 3000
+  save_iters: 3000
+
+optimizer:
+  type: Adam
+  true_wd: False
+  wd: 0.0
+  bn_wd: False
+  clip_grad: 20
+  lr: 0.0001
+  args: {
+    betas: !!python/tuple [0.9, 0.999], # for default Adam 
+  }
+  scheduler: {
+    periods: [6, 2],
+    gamma: 0.1,
+  }
+
+model:
+  name: 'modules.model_vision.BaseVision'
+  checkpoint: ~
+  vision: {
+    loss_weight: 1.,
+    attention: 'position',
+    backbone: 'transformer',
+    backbone_ln: 3,
+  }

configs/pretrain_vision_model_sv.yaml

@@ -0,0 +1,58 @@
+global:
+  name: pretrain-vision-model-sv
+  phase: train
+  stage: pretrain-vision
+  workdir: workdir
+  seed: ~
+ 
+dataset:
+  train: {
+    roots: ['data/training/MJ/MJ_train/', 
+            'data/training/MJ/MJ_test/', 
+            'data/training/MJ/MJ_valid/', 
+            'data/training/ST'],
+    batch_size: 384
+  }
+  test: {
+    roots: ['data/evaluation/IIIT5k_3000', 
+            'data/evaluation/SVT', 
+            'data/evaluation/SVTP',
+            'data/evaluation/IC13_857',
+            'data/evaluation/IC15_1811',
+            'data/evaluation/CUTE80'],
+    batch_size: 384
+  }
+  data_aug: True
+  multiscales: False
+  num_workers: 14
+
+training:
+  epochs: 8
+  show_iters: 50
+  eval_iters: 3000
+  save_iters: 3000
+
+optimizer:
+  type: Adam
+  true_wd: False
+  wd: 0.0
+  bn_wd: False
+  clip_grad: 20
+  lr: 0.0001
+  args: {
+    betas: !!python/tuple [0.9, 0.999], # for default Adam 
+  }
+  scheduler: {
+    periods: [6, 2],
+    gamma: 0.1,
+  }
+
+model:
+  name: 'modules.model_vision.BaseVision'
+  checkpoint: ~
+  vision: {
+    loss_weight: 1.,
+    attention: 'attention',
+    backbone: 'transformer',
+    backbone_ln: 2,
+  }

configs/template.yaml

@@ -0,0 +1,67 @@
+global:
+  name: exp
+  phase: train
+  stage: pretrain-vision
+  workdir: /tmp/workdir
+  seed: ~
+ 
+dataset:
+  train: {
+    roots: ['data/training/MJ/MJ_train/', 
+            'data/training/MJ/MJ_test/', 
+            'data/training/MJ/MJ_valid/', 
+            'data/training/ST'],
+    batch_size: 128
+  }
+  test: {
+    roots: ['data/evaluation/IIIT5k_3000', 
+            'data/evaluation/SVT', 
+            'data/evaluation/SVTP',
+            'data/evaluation/IC13_857',
+            'data/evaluation/IC15_1811',
+            'data/evaluation/CUTE80'],
+    batch_size: 128
+  }
+  charset_path: data/charset_36.txt
+  num_workers: 4
+  max_length: 25  # 30
+  image_height: 32
+  image_width: 128
+  case_sensitive: False
+  eval_case_sensitive: False
+  data_aug: True
+  multiscales: False
+  pin_memory: True
+  smooth_label: False
+  smooth_factor: 0.1
+  one_hot_y: True
+  use_sm: False
+
+training:
+  epochs: 6
+  show_iters: 50
+  eval_iters: 3000
+  save_iters: 20000
+  start_iters: 0
+  stats_iters: 100000
+
+optimizer:
+  type: Adadelta # Adadelta, Adam
+  true_wd: False
+  wd: 0. # 0.001
+  bn_wd: False
+  args: {
+    # betas: !!python/tuple [0.9, 0.99], # betas=(0.9,0.99) for AdamW
+    # betas: !!python/tuple [0.9, 0.999], # for default Adam 
+  }
+  clip_grad: 20
+  lr: [1.0, 1.0, 1.0]  # lr: [0.005, 0.005, 0.005]   
+  scheduler: {
+    periods: [3, 2, 1],
+    gamma: 0.1,
+  }
+
+model:
+  name: 'modules.model_abinet.ABINetModel'
+  checkpoint: ~
+  strict: True

configs/train_abinet.yaml

@@ -0,0 +1,71 @@
+global:
+  name: train-abinet
+  phase: train
+  stage: train-super
+  workdir: workdir
+  seed: ~
+ 
+dataset:
+  train: {
+    roots: ['data/training/MJ/MJ_train/', 
+            'data/training/MJ/MJ_test/', 
+            'data/training/MJ/MJ_valid/', 
+            'data/training/ST'],
+    batch_size: 384
+  }
+  test: {
+    roots: ['data/evaluation/IIIT5k_3000', 
+            'data/evaluation/SVT', 
+            'data/evaluation/SVTP',
+            'data/evaluation/IC13_857',
+            'data/evaluation/IC15_1811',
+            'data/evaluation/CUTE80'],
+    batch_size: 384
+  }
+  data_aug: True
+  multiscales: False
+  num_workers: 14
+
+training:
+  epochs: 10
+  show_iters: 50
+  eval_iters: 3000
+  save_iters: 3000
+
+optimizer:
+  type: Adam
+  true_wd: False
+  wd: 0.0
+  bn_wd: False
+  clip_grad: 20
+  lr: 0.0001
+  args: {
+    betas: !!python/tuple [0.9, 0.999], # for default Adam 
+  }
+  scheduler: {
+    periods: [6, 4],
+    gamma: 0.1,
+  }
+
+model:
+  name: 'modules.model_abinet_iter.ABINetIterModel'
+  iter_size: 3
+  ensemble: ''
+  use_vision: False
+  vision: {
+    checkpoint: workdir/pretrain-vision-model/best-pretrain-vision-model.pth,
+    loss_weight: 1.,
+    attention: 'position',
+    backbone: 'transformer',
+    backbone_ln: 3,
+  }
+  language: {
+    checkpoint:  workdir/pretrain-language-model/pretrain-language-model.pth,
+    num_layers: 4,
+    loss_weight: 1.,
+    detach: True,
+    use_self_attn: False
+  }
+  alignment: {
+    loss_weight: 1.,
+  }

configs/train_abinet_sv.yaml

@@ -0,0 +1,71 @@
+global:
+  name: train-abinet-sv
+  phase: train
+  stage: train-super
+  workdir: workdir
+  seed: ~
+ 
+dataset:
+  train: {
+    roots: ['data/training/MJ/MJ_train/', 
+            'data/training/MJ/MJ_test/', 
+            'data/training/MJ/MJ_valid/', 
+            'data/training/ST'],
+    batch_size: 384
+  }
+  test: {
+    roots: ['data/evaluation/IIIT5k_3000', 
+            'data/evaluation/SVT', 
+            'data/evaluation/SVTP',
+            'data/evaluation/IC13_857',
+            'data/evaluation/IC15_1811',
+            'data/evaluation/CUTE80'],
+    batch_size: 384
+  }
+  data_aug: True
+  multiscales: False
+  num_workers: 14
+
+training:
+  epochs: 10
+  show_iters: 50
+  eval_iters: 3000
+  save_iters: 3000
+
+optimizer:
+  type: Adam
+  true_wd: False
+  wd: 0.0
+  bn_wd: False
+  clip_grad: 20
+  lr: 0.0001
+  args: {
+    betas: !!python/tuple [0.9, 0.999], # for default Adam 
+  }
+  scheduler: {
+    periods: [6, 4],
+    gamma: 0.1,
+  }
+
+model:
+  name: 'modules.model_abinet_iter.ABINetIterModel'
+  iter_size: 3
+  ensemble: ''
+  use_vision: False
+  vision: {
+    checkpoint: workdir/pretrain-vision-model-sv/best-pretrain-vision-model-sv.pth,
+    loss_weight: 1.,
+    attention: 'attention',
+    backbone: 'transformer',
+    backbone_ln: 2,
+  }
+  language: {
+    checkpoint:  workdir/pretrain-language-model/pretrain-language-model.pth,
+    num_layers: 4,
+    loss_weight: 1.,
+    detach: True,
+    use_self_attn: False
+  }
+  alignment: {
+    loss_weight: 1.,
+  }

configs/train_abinet_wo_iter.yaml

@@ -0,0 +1,68 @@
+global:
+  name: train-abinet-wo-iter
+  phase: train
+  stage: train-super
+  workdir: workdir
+  seed: ~
+ 
+dataset:
+  train: {
+    roots: ['data/training/MJ/MJ_train/', 
+            'data/training/MJ/MJ_test/', 
+            'data/training/MJ/MJ_valid/', 
+            'data/training/ST'],
+    batch_size: 384
+  }
+  test: {
+    roots: ['data/evaluation/IIIT5k_3000', 
+            'data/evaluation/SVT', 
+            'data/evaluation/SVTP',
+            'data/evaluation/IC13_857',
+            'data/evaluation/IC15_1811',
+            'data/evaluation/CUTE80'],
+    batch_size: 384
+  }
+  data_aug: True
+  multiscales: False
+  num_workers: 14
+
+training:
+  epochs: 10
+  show_iters: 50
+  eval_iters: 3000
+  save_iters: 3000
+
+optimizer:
+  type: Adam
+  true_wd: False
+  wd: 0.0
+  bn_wd: False
+  clip_grad: 20
+  lr: 0.0001
+  args: {
+    betas: !!python/tuple [0.9, 0.999], # for default Adam 
+  }
+  scheduler: {
+    periods: [6, 4],
+    gamma: 0.1,
+  }
+
+model:
+  name: 'modules.model_abinet.ABINetModel'
+  vision: {
+    checkpoint: workdir/pretrain-vision-model/best-pretrain-vision-model.pth,
+    loss_weight: 1.,
+    attention: 'position',
+    backbone: 'transformer',
+    backbone_ln: 3,
+  }
+  language: {
+    checkpoint:  workdir/pretrain-language-model/pretrain-language-model.pth,
+    num_layers: 4,
+    loss_weight: 1.,
+    detach: True,
+    use_self_attn: False
+  }
+  alignment: {
+    loss_weight: 1.,
+  }