From 16c93c20f4e1106f6480e078f5a8fabdcc4090d2 Mon Sep 17 00:00:00 2001 From: Yanghao Li Date: Wed, 13 Jul 2022 11:51:49 -0700 Subject: [PATCH 01/34] fix links Reviewed By: haooooooqi Differential Revision: D37320605 fbshipit-source-id: eb8fe671f38c1c2cea29067a0ea14b99fbffc050 --- projects/mvitv2/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/mvitv2/README.md b/projects/mvitv2/README.md index 9e2e0fa0f..3589784fc 100644 --- a/projects/mvitv2/README.md +++ b/projects/mvitv2/README.md @@ -35,7 +35,7 @@ Here we can train a standard MViTv2 model from scratch by: ``` python tools/run_net.py \ - --cfg configs/Kinetics/MVIT-B.yaml \ + --cfg configs/Kinetics/MVITv2_S_16x4.yaml \ DATA.PATH_TO_DATA_DIR path_to_your_dataset \ ``` From fee19d699c49a81f33b890c5ff592bbb11aa5c54 Mon Sep 17 00:00:00 2001 From: Yanghao Li Date: Thu, 14 Jul 2022 11:48:19 -0700 Subject: [PATCH 02/34] add mvitv2 for IN models Reviewed By: feichtenhofer, haooooooqi Differential Revision: D37659333 fbshipit-source-id: 60a96aab4e36efe0743b1ba1b3067c0278e3c8cb --- configs/ImageNet/MVITv2_B.yaml | 88 ++++++++++++++++++++++++++++++++++ configs/ImageNet/MVITv2_S.yaml | 88 ++++++++++++++++++++++++++++++++++ configs/ImageNet/MVITv2_T.yaml | 88 ++++++++++++++++++++++++++++++++++ projects/mvitv2/README.md | 11 +++++ slowfast/datasets/imagenet.py | 6 ++- 5 files changed, 279 insertions(+), 2 deletions(-) create mode 100644 configs/ImageNet/MVITv2_B.yaml create mode 100644 configs/ImageNet/MVITv2_S.yaml create mode 100644 configs/ImageNet/MVITv2_T.yaml diff --git a/configs/ImageNet/MVITv2_B.yaml b/configs/ImageNet/MVITv2_B.yaml new file mode 100644 index 000000000..c6aea74bd --- /dev/null +++ b/configs/ImageNet/MVITv2_B.yaml @@ -0,0 +1,88 @@ +TRAIN: + ENABLE: True + DATASET: imagenet + BATCH_SIZE: 256 + EVAL_PERIOD: 10 + CHECKPOINT_PERIOD: 1 + AUTO_RESUME: True +DATA: + # PATH_TO_DATA_DIR: path-to-imagenet-dir + MEAN: [0.485, 0.456, 0.406] + STD: [0.229, 0.224, 0.225] + NUM_FRAMES: 1 + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 224 + INPUT_CHANNEL_NUM: [3] +MVIT: + PATCH_2D: True + ZERO_DECAY_POS_CLS: False + MODE: "conv" + CLS_EMBED_ON: False + PATCH_KERNEL: [7, 7] + PATCH_STRIDE: [4, 4] + PATCH_PADDING: [3, 3] + EMBED_DIM: 96 + NUM_HEADS: 1 + MLP_RATIO: 4.0 + QKV_BIAS: True +DROPPATH_RATE: 0.3 + DEPTH: 24 + NORM: "layernorm" + DIM_MUL: [[2, 2.0], [5, 2.0], [21, 2.0]] + HEAD_MUL: [[2, 2.0], [5, 2.0], [21, 2.0]] + POOL_KVQ_KERNEL: [1, 3, 3] + POOL_KV_STRIDE_ADAPTIVE: [1, 4, 4] + POOL_Q_STRIDE: [[0, 1, 1, 1], [1, 1, 1, 1], [2, 1, 2, 2], [3, 1, 1, 1], [4, 1, 1, 1], [5, 1, 2, 2], [6, 1, 1, 1], [7, 1, 1, 1], [8, 1, 1, 1], [9, 1, 1, 1], [10, 1, 1, 1], [11, 1, 1, 1], [12, 1, 1, 1], [13, 1, 1, 1], [14, 1, 1, 1], [15, 1, 1, 1], [16, 1, 1, 1], [17, 1, 1, 1], [18, 1, 1, 1], [19, 1, 1, 1], [20, 1, 1, 1], [21, 1, 2, 2], [22, 1, 1, 1], [23, 1, 1, 1]] + RESIDUAL_POOLING: True + USE_ABS_POS: False + REL_POS_SPATIAL: True + DIM_MUL_IN_ATT: True +AUG: + ENABLE: True + COLOR_JITTER: 0.4 + AA_TYPE: rand-m9-n6-mstd0.5-inc1 + INTERPOLATION: bicubic + RE_PROB: 0.25 + RE_MODE: pixel + RE_COUNT: 1 + RE_SPLIT: False +MIXUP: + ENABLE: True + ALPHA: 0.8 + CUTMIX_ALPHA: 1.0 + PROB: 1.0 + SWITCH_PROB: 0.5 + LABEL_SMOOTH_VALUE: 0.1 +SOLVER: + BASE_LR_SCALE_NUM_SHARDS: True + BASE_LR: 0.00025 + LR_POLICY: cosine + MAX_EPOCH: 300 + MOMENTUM: 0.9 + WEIGHT_DECAY: 0.05 + WARMUP_EPOCHS: 70.0 + WARMUP_START_LR: 1e-8 + OPTIMIZING_METHOD: adamw + COSINE_AFTER_WARMUP: True + COSINE_END_LR: 1e-6 + ZERO_WD_1D_PARAM: True + CLIP_GRAD_L2NORM: 1.0 +MODEL: + NUM_CLASSES: 1000 + ARCH: mvit + MODEL_NAME: MViT + LOSS_FUNC: soft_cross_entropy + DROPOUT_RATE: 0.0 +TEST: + ENABLE: True + DATASET: imagenet + BATCH_SIZE: 256 + NUM_ENSEMBLE_VIEWS: 1 + NUM_SPATIAL_CROPS: 1 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git a/configs/ImageNet/MVITv2_S.yaml b/configs/ImageNet/MVITv2_S.yaml new file mode 100644 index 000000000..ec5ec7eb2 --- /dev/null +++ b/configs/ImageNet/MVITv2_S.yaml @@ -0,0 +1,88 @@ +TRAIN: + ENABLE: True + DATASET: imagenet + BATCH_SIZE: 256 + EVAL_PERIOD: 10 + CHECKPOINT_PERIOD: 1 + AUTO_RESUME: True +DATA: + # PATH_TO_DATA_DIR: path-to-imagenet-dir + MEAN: [0.485, 0.456, 0.406] + STD: [0.229, 0.224, 0.225] + NUM_FRAMES: 1 + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 224 + INPUT_CHANNEL_NUM: [3] +MVIT: + PATCH_2D: True + ZERO_DECAY_POS_CLS: False + MODE: "conv" + CLS_EMBED_ON: False + PATCH_KERNEL: [7, 7] + PATCH_STRIDE: [4, 4] + PATCH_PADDING: [3, 3] + EMBED_DIM: 96 + NUM_HEADS: 1 + MLP_RATIO: 4.0 + QKV_BIAS: True + DROPPATH_RATE: 0.1 + DEPTH: 16 + NORM: "layernorm" + DIM_MUL: [[1, 2.0], [3, 2.0], [14, 2.0]] + HEAD_MUL: [[1, 2.0], [3, 2.0], [14, 2.0]] + POOL_KVQ_KERNEL: [1, 3, 3] + POOL_KV_STRIDE: [[0, 1, 4, 4], [1, 1, 2, 2], [2, 1, 2, 2], [3, 1, 1, 1], [4, 1, 1, 1], [5, 1, 1, 1], [6, 1, 1, 1], [7, 1, 1, 1], [8, 1, 1, 1], [9, 1, 1, 1], [10, 1, 1, 1], [11, 1, 1, 1], [12, 1, 1, 1], [13, 1, 1, 1], [14, 1, 1, 1], [15, 1, 1, 1]] + POOL_Q_STRIDE: [[0, 1, 1, 1], [1, 1, 2, 2], [2, 1, 1, 1], [3, 1, 2, 2], [4, 1, 1, 1], [5, 1, 1, 1], [6, 1, 1, 1], [7, 1, 1, 1], [8, 1, 1, 1], [9, 1, 1, 1], [10, 1, 1, 1], [11, 1, 1, 1], [12, 1, 1, 1], [13, 1, 1, 1], [14, 1, 2, 2], [15, 1, 1, 1]] + RESIDUAL_POOLING: True + USE_ABS_POS: False + REL_POS_SPATIAL: True + DIM_MUL_IN_ATT: True +AUG: + ENABLE: True + COLOR_JITTER: 0.4 + AA_TYPE: rand-m9-n6-mstd0.5-inc1 + INTERPOLATION: bicubic + RE_PROB: 0.25 + RE_MODE: pixel + RE_COUNT: 1 + RE_SPLIT: False +MIXUP: + ENABLE: True + ALPHA: 0.8 + CUTMIX_ALPHA: 1.0 + PROB: 1.0 + SWITCH_PROB: 0.5 + LABEL_SMOOTH_VALUE: 0.1 +SOLVER: + BASE_LR_SCALE_NUM_SHARDS: True + BASE_LR: 0.00025 + LR_POLICY: cosine + MAX_EPOCH: 300 + MOMENTUM: 0.9 + WEIGHT_DECAY: 0.05 + WARMUP_EPOCHS: 70.0 + WARMUP_START_LR: 1e-8 + OPTIMIZING_METHOD: adamw + COSINE_AFTER_WARMUP: True + COSINE_END_LR: 1e-6 + ZERO_WD_1D_PARAM: True + CLIP_GRAD_L2NORM: 1.0 +MODEL: + NUM_CLASSES: 1000 + ARCH: mvit + MODEL_NAME: MViT + LOSS_FUNC: soft_cross_entropy + DROPOUT_RATE: 0.0 +TEST: + ENABLE: True + DATASET: imagenet + BATCH_SIZE: 256 + NUM_ENSEMBLE_VIEWS: 1 + NUM_SPATIAL_CROPS: 1 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git a/configs/ImageNet/MVITv2_T.yaml b/configs/ImageNet/MVITv2_T.yaml new file mode 100644 index 000000000..6cb7989fe --- /dev/null +++ b/configs/ImageNet/MVITv2_T.yaml @@ -0,0 +1,88 @@ +TRAIN: + ENABLE: True + DATASET: imagenet + BATCH_SIZE: 256 + EVAL_PERIOD: 10 + CHECKPOINT_PERIOD: 1 + AUTO_RESUME: True +DATA: + # PATH_TO_DATA_DIR: path-to-imagenet-dir + MEAN: [0.485, 0.456, 0.406] + STD: [0.229, 0.224, 0.225] + NUM_FRAMES: 1 + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 224 + INPUT_CHANNEL_NUM: [3] +MVIT: + PATCH_2D: True + ZERO_DECAY_POS_CLS: False + MODE: "conv" + CLS_EMBED_ON: False + PATCH_KERNEL: [7, 7] + PATCH_STRIDE: [4, 4] + PATCH_PADDING: [3, 3] + EMBED_DIM: 96 + NUM_HEADS: 1 + MLP_RATIO: 4.0 + QKV_BIAS: True + DROPPATH_RATE: 0.1 + DEPTH: 10 + NORM: "layernorm" + DIM_MUL: [[1, 2.0], [3, 2.0], [8, 2.0]] + HEAD_MUL: [[1, 2.0], [3, 2.0], [8, 2.0]] + POOL_KVQ_KERNEL: [1, 3, 3] + POOL_KV_STRIDE: [[0, 1, 4, 4], [1, 1, 2, 2], [2, 1, 2, 2], [3, 1, 1, 1], [4, 1, 1, 1], [5, 1, 1, 1], [6, 1, 1, 1], [7, 1, 1, 1], [8, 1, 1, 1], [9, 1, 1, 1]] + POOL_Q_STRIDE: [[0, 1, 1, 1], [1, 1, 2, 2], [2, 1, 1, 1], [3, 1, 2, 2], [4, 1, 1, 1], [5, 1, 1, 1], [6, 1, 1, 1], [7, 1, 1, 1], [8, 1, 2, 2], [9, 1, 1, 1]] + RESIDUAL_POOLING: True + USE_ABS_POS: False + REL_POS_SPATIAL: True + DIM_MUL_IN_ATT: True +AUG: + ENABLE: True + COLOR_JITTER: 0.4 + AA_TYPE: rand-m9-n6-mstd0.5-inc1 + INTERPOLATION: bicubic + RE_PROB: 0.25 + RE_MODE: pixel + RE_COUNT: 1 + RE_SPLIT: False +MIXUP: + ENABLE: True + ALPHA: 0.8 + CUTMIX_ALPHA: 1.0 + PROB: 1.0 + SWITCH_PROB: 0.5 + LABEL_SMOOTH_VALUE: 0.1 +SOLVER: + BASE_LR_SCALE_NUM_SHARDS: True + BASE_LR: 0.00025 + LR_POLICY: cosine + MAX_EPOCH: 300 + MOMENTUM: 0.9 + WEIGHT_DECAY: 0.05 + WARMUP_EPOCHS: 70.0 + WARMUP_START_LR: 1e-8 + OPTIMIZING_METHOD: adamw + COSINE_AFTER_WARMUP: True + COSINE_END_LR: 1e-6 + ZERO_WD_1D_PARAM: True + CLIP_GRAD_L2NORM: 1.0 +MODEL: + NUM_CLASSES: 1000 + ARCH: mvit + MODEL_NAME: MViT + LOSS_FUNC: soft_cross_entropy + DROPOUT_RATE: 0.0 +TEST: + ENABLE: True + DATASET: imagenet + BATCH_SIZE: 256 + NUM_ENSEMBLE_VIEWS: 1 + NUM_SPATIAL_CROPS: 1 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git a/projects/mvitv2/README.md b/projects/mvitv2/README.md index 3589784fc..ad110648e 100644 --- a/projects/mvitv2/README.md +++ b/projects/mvitv2/README.md @@ -29,6 +29,17 @@ MViT is a multiscale transformer which serves as a general vision backbone for d | MViTv2-B | 32 x 3 | 82.9 | 95.7 | 225 x 1 x 5 | 51.2 | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/model_zoo/mvitv2/pysf_video_models/MViTv2_B_32x3_k400_f304025456.pyth) | Kinetics/MVITv2_B_32x3 | | MViTv2-L | 40 x 3 | 86.1 | 97.0 | 2828 x 3 x 5 | 217.6 | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/model_zoo/mvitv2/pysf_video_models/MViTv2_L_40x3_k400_f306903192.pyth) | Kinetics/MVITv2_L_40x3_test | + +### ImageNet-1K + +| name | resolution |acc@1 | #params | FLOPs | 1k model | +|:---:|:---:|:---:|:---:| :---:|:---:| +| MViTv2-T | 224x224 | 82.3 | 24M | 4.7G | [model](https://dl.fbaipublicfiles.com/mvit/mvitv2_models/MViTv2_T_in1k.pyth) | +| MViTv2-S | 224x224 | 83.6 | 35M | 7.0G | [model](https://dl.fbaipublicfiles.com/mvit/mvitv2_models/MViTv2_S_in1k.pyth) | +| MViTv2-B | 224x224 | 84.4 | 52M | 10.2G | [model](https://dl.fbaipublicfiles.com/mvit/mvitv2_models/MViTv2_B_in1k.pyth) | + +For more ImageNet results, please check the MViTv2 for image classification [repo](https://github.com/facebookresearch/mvit). + ## Get started Here we can train a standard MViTv2 model from scratch by: diff --git a/slowfast/datasets/imagenet.py b/slowfast/datasets/imagenet.py index 1a706621d..5ad6428f5 100644 --- a/slowfast/datasets/imagenet.py +++ b/slowfast/datasets/imagenet.py @@ -41,6 +41,7 @@ def __init__(self, cfg, mode, num_retries=10): self._construct_imdb() else: self._load_imdb() + self.num_videos = len(self._imdb) def _load_imdb(self): split_path = os.path.join( @@ -194,10 +195,11 @@ def __getitem__(self, index): if isinstance(im, list): label = [label for _ in range(len(im))] dummy = [torch.Tensor() for _ in range(len(im))] - return im, label, dummy, dummy, {} + return im, label, index, dummy, {} else: dummy = torch.Tensor() - return [im], label, dummy, dummy, {} + return [im], label, index, dummy, {} + def __len__(self): return len(self._imdb) From adc55ca38a62a8bf3899a2d7c2dfbc9f9eee1ade Mon Sep 17 00:00:00 2001 From: Christoph Feichtenhofer Date: Wed, 3 Aug 2022 19:36:22 -0700 Subject: [PATCH 03/34] maskfeat + MAE init Reviewed By: haooooooqi, lyttonhao, weichen582 Differential Revision: D37582534 fbshipit-source-id: 4028c1f922bb9453794caa98efd1f380321378b8 --- MODEL_ZOO.md | 4 +- README.md | 9 +- .../BYOL_SlowR50_8x8.yaml | 2 +- .../MoCo_SlowR50_8x8.yaml | 2 +- .../SimCLR_SlowR50_8x8.yaml | 0 .../SwAV_Slow_R50_8x8.yaml | 0 .../finetune_SSv2_Slow_R50_syn0.yaml | 0 .../finetune_SSv2_Slow_R50_syn8.yaml | 0 .../finetune_ava_Slow_R50_syn0.yaml | 0 .../finetune_ava_Slow_R50_syn8.yaml | 0 .../finetune_ucf_Slow_R50_syn0.yaml | 0 .../finetune_ucf_Slow_R50_syn8.yaml | 0 .../linear_k400_Slow_8x8_R50_syn0.yaml | 0 .../linear_k400_Slow_8x8_R50_syn8.yaml | 0 .../masked_ssl/in1k_VIT_B_MaskFeat_FT.yaml | 84 +++ .../masked_ssl/in1k_VIT_B_MaskFeat_PT.yaml | 80 +++ .../masked_ssl/in1k_VIT_L_MaskFeat_FT.yaml | 84 +++ .../masked_ssl/in1k_VIT_L_MaskFeat_PT.yaml | 80 +++ configs/masked_ssl/k400_MVITv2_L_16x4_FT.yaml | 108 +++ .../k400_MVITv2_L_16x4_MaskFeat_PT.yaml | 105 +++ configs/masked_ssl/k400_MVITv2_S_16x4_FT.yaml | 101 +++ .../k400_MVITv2_S_16x4_MaskFeat_PT.yaml | 98 +++ configs/masked_ssl/k400_VIT_B_16x4_FT.yaml | 90 +++ .../masked_ssl/k400_VIT_B_16x4_MAE_PT.yaml | 86 +++ configs/masked_ssl/k400_VIT_H_16x4_FT.yaml | 88 +++ .../masked_ssl/k400_VIT_H_16x4_MAE_PT.yaml | 88 +++ configs/masked_ssl/k400_VIT_L_16x4_FT.yaml | 89 +++ .../masked_ssl/k400_VIT_L_16x4_MAE_PT.yaml | 87 +++ projects/{ssl => contrastive_ssl}/README.md | 20 +- projects/mae/README.md | 53 ++ projects/maskfeat/README.md | 81 +++ slowfast/config/defaults.py | 118 +++- slowfast/datasets/decoder.py | 24 +- slowfast/datasets/imagenet.py | 76 +- slowfast/datasets/kinetics.py | 81 ++- slowfast/datasets/loader.py | 11 +- slowfast/datasets/mixup.py | 9 +- slowfast/datasets/transform.py | 186 +++++ slowfast/models/__init__.py | 3 +- slowfast/models/attention.py | 33 +- slowfast/models/contrastive.py | 79 ++- slowfast/models/head_helper.py | 118 ++++ slowfast/models/losses.py | 44 +- slowfast/models/masked.py | 658 ++++++++++++++++++ slowfast/models/operators.py | 65 ++ slowfast/models/optimizer.py | 268 +++++-- slowfast/models/stem_helper.py | 5 +- slowfast/models/utils.py | 196 ++++++ slowfast/models/video_model_builder.py | 131 +++- slowfast/utils/benchmark.py | 0 slowfast/utils/checkpoint.py | 216 +++++- slowfast/utils/logging.py | 4 +- slowfast/utils/meters.py | 80 ++- slowfast/utils/misc.py | 135 +++- tools/benchmark.py | 0 tools/run_net.py | 9 +- tools/test_net.py | 175 +++-- tools/train_net.py | 184 +++-- 58 files changed, 3989 insertions(+), 358 deletions(-) rename configs/{ssl => contrastive_ssl}/BYOL_SlowR50_8x8.yaml (98%) rename configs/{ssl => contrastive_ssl}/MoCo_SlowR50_8x8.yaml (98%) rename configs/{ssl => contrastive_ssl}/SimCLR_SlowR50_8x8.yaml (100%) rename configs/{ssl => contrastive_ssl}/SwAV_Slow_R50_8x8.yaml (100%) rename configs/{ssl => contrastive_ssl}/finetune_SSv2_Slow_R50_syn0.yaml (100%) rename configs/{ssl => contrastive_ssl}/finetune_SSv2_Slow_R50_syn8.yaml (100%) rename configs/{ssl => contrastive_ssl}/finetune_ava_Slow_R50_syn0.yaml (100%) rename configs/{ssl => contrastive_ssl}/finetune_ava_Slow_R50_syn8.yaml (100%) rename configs/{ssl => contrastive_ssl}/finetune_ucf_Slow_R50_syn0.yaml (100%) rename configs/{ssl => contrastive_ssl}/finetune_ucf_Slow_R50_syn8.yaml (100%) rename configs/{ssl => contrastive_ssl}/linear_k400_Slow_8x8_R50_syn0.yaml (100%) rename configs/{ssl => contrastive_ssl}/linear_k400_Slow_8x8_R50_syn8.yaml (100%) create mode 100644 configs/masked_ssl/in1k_VIT_B_MaskFeat_FT.yaml create mode 100644 configs/masked_ssl/in1k_VIT_B_MaskFeat_PT.yaml create mode 100644 configs/masked_ssl/in1k_VIT_L_MaskFeat_FT.yaml create mode 100644 configs/masked_ssl/in1k_VIT_L_MaskFeat_PT.yaml create mode 100644 configs/masked_ssl/k400_MVITv2_L_16x4_FT.yaml create mode 100644 configs/masked_ssl/k400_MVITv2_L_16x4_MaskFeat_PT.yaml create mode 100644 configs/masked_ssl/k400_MVITv2_S_16x4_FT.yaml create mode 100644 configs/masked_ssl/k400_MVITv2_S_16x4_MaskFeat_PT.yaml create mode 100644 configs/masked_ssl/k400_VIT_B_16x4_FT.yaml create mode 100644 configs/masked_ssl/k400_VIT_B_16x4_MAE_PT.yaml create mode 100644 configs/masked_ssl/k400_VIT_H_16x4_FT.yaml create mode 100644 configs/masked_ssl/k400_VIT_H_16x4_MAE_PT.yaml create mode 100644 configs/masked_ssl/k400_VIT_L_16x4_FT.yaml create mode 100644 configs/masked_ssl/k400_VIT_L_16x4_MAE_PT.yaml rename projects/{ssl => contrastive_ssl}/README.md (73%) create mode 100644 projects/mae/README.md create mode 100644 projects/maskfeat/README.md create mode 100644 slowfast/models/masked.py mode change 100755 => 100644 slowfast/utils/benchmark.py mode change 100755 => 100644 tools/benchmark.py diff --git a/MODEL_ZOO.md b/MODEL_ZOO.md index 0436dae72..8971c6a0a 100644 --- a/MODEL_ZOO.md +++ b/MODEL_ZOO.md @@ -68,8 +68,8 @@ We also release the imagenet pretrained model if finetuning from ImageNet is pre | architecture | size | Top1 | Top5 | model | Config | | ------------- | ------------- | ------------- | ------------- | ------------- | ------------- | -| ResNet | R50 | 23.6 | 6.8 | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/model_zoo/kinetics400/R50_IN1K.pyth) | ImageNet/RES_R50 | -| MVIT | B-16-Conv | 17.1 | 3.7 | [`link`](https://drive.google.com/file/d/1dYYqUB-3DSgBVc9d6o-rW8ojtVsrFLgp/view?usp=sharing) | ImageNet/MVIT_B_16_CONV | +| ResNet | R50 | 73.7 | 93.2 | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/model_zoo/kinetics400/R50_IN1K.pyth) | ImageNet/RES_R50 | +| MVIT | B-16-Conv | 82.9 | 96.3 | [`link`](https://drive.google.com/file/d/1dYYqUB-3DSgBVc9d6o-rW8ojtVsrFLgp/view?usp=sharing) | ImageNet/MVIT_B_16_CONV | diff --git a/README.md b/README.md index fcd49dbde..53a03c6b9 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,11 @@ PySlowFast is an open source video understanding codebase from FAIR that provide - [Non-local Neural Networks](https://arxiv.org/abs/1711.07971) - [A Multigrid Method for Efficiently Training Video Models](https://arxiv.org/abs/1912.00998) - [X3D: Progressive Network Expansion for Efficient Video Recognition](https://arxiv.org/abs/2004.04730) -- [Multiscale Vision Transformers](https://arxiv.org/abs/2104.11227.pdf) +- [Multiscale Vision Transformers](https://arxiv.org/abs/2104.11227) +- [A Large-Scale Study on Unsupervised Spatiotemporal Representation Learning](https://arxiv.org/abs/2104.14558) +- [MViTv2: Improved Multiscale Vision Transformers for Classification and Detection](https://arxiv.org/abs/2112.01526) +- [Masked Feature Prediction for Self-Supervised Visual Pre-Training](https://arxiv.org/abs/2112.09133) +- [Masked Autoencoders As Spatiotemporal Learners](https://arxiv.org/abs/2205.09113)
@@ -25,7 +29,10 @@ The goal of PySlowFast is to provide a high-performance, light-weight pytorch co - MViTv1 and MViTv2 ## Updates + - We now support [MAE for Video](https://arxiv.org/abs/2104.11227.pdf). See [`projects/mae`](./projects/mae/README.md) for more information. + - We now support [MaskFeat](https://arxiv.org/abs/2112.09133). See [`projects/maskfeat`](./projects/maskfeat/README.md) for more information. - We now support [MViTv2](https://arxiv.org/abs/2104.11227.pdf) in PySlowFast. See [`projects/mvitv2`](./projects/mvitv2/README.md) for more information. + - We now support [A Large-Scale Study on Unsupervised Spatiotemporal Representation Learning](https://arxiv.org/abs/2104.14558). See [`projects/contrastive_ssl`](./projects/contrastive_ssl/README.md) for more information. - We now support [Multiscale Vision Transformers](https://arxiv.org/abs/2104.11227.pdf) on Kinetics and ImageNet. See [`projects/mvit`](./projects/mvit/README.md) for more information. - We now support [PyTorchVideo](https://github.com/facebookresearch/pytorchvideo) models and datasets. See [`projects/pytorchvideo`](./projects/pytorchvideo/README.md) for more information. - We now support [X3D Models](https://arxiv.org/abs/2004.04730). See [`projects/x3d`](./projects/x3d/README.md) for more information. diff --git a/configs/ssl/BYOL_SlowR50_8x8.yaml b/configs/contrastive_ssl/BYOL_SlowR50_8x8.yaml similarity index 98% rename from configs/ssl/BYOL_SlowR50_8x8.yaml rename to configs/contrastive_ssl/BYOL_SlowR50_8x8.yaml index 0570f0fe2..64efa45df 100644 --- a/configs/ssl/BYOL_SlowR50_8x8.yaml +++ b/configs/contrastive_ssl/BYOL_SlowR50_8x8.yaml @@ -13,7 +13,7 @@ MODEL: DROPOUT_RATE: 0.0 HEAD_ACT: none CONTRASTIVE: - T: 0.5 # default 0.07 + T: 0.5 DIM: 256 # 128 default, if changed, change nCls too NUM_MLP_LAYERS: 2 # default 1 BN_MLP: True diff --git a/configs/ssl/MoCo_SlowR50_8x8.yaml b/configs/contrastive_ssl/MoCo_SlowR50_8x8.yaml similarity index 98% rename from configs/ssl/MoCo_SlowR50_8x8.yaml rename to configs/contrastive_ssl/MoCo_SlowR50_8x8.yaml index 8fb40e07e..09879627d 100644 --- a/configs/ssl/MoCo_SlowR50_8x8.yaml +++ b/configs/contrastive_ssl/MoCo_SlowR50_8x8.yaml @@ -15,7 +15,7 @@ MODEL: DROPOUT_RATE: 0.0 HEAD_ACT: none CONTRASTIVE: - T: 0.1 # default 0.07 + T: 0.1 NUM_MLP_LAYERS: 3 # default 1 SEQUENTIAL: True # def fault MOCO_MULTI_VIEW_QUEUE: True # default: False diff --git a/configs/ssl/SimCLR_SlowR50_8x8.yaml b/configs/contrastive_ssl/SimCLR_SlowR50_8x8.yaml similarity index 100% rename from configs/ssl/SimCLR_SlowR50_8x8.yaml rename to configs/contrastive_ssl/SimCLR_SlowR50_8x8.yaml diff --git a/configs/ssl/SwAV_Slow_R50_8x8.yaml b/configs/contrastive_ssl/SwAV_Slow_R50_8x8.yaml similarity index 100% rename from configs/ssl/SwAV_Slow_R50_8x8.yaml rename to configs/contrastive_ssl/SwAV_Slow_R50_8x8.yaml diff --git a/configs/ssl/finetune_SSv2_Slow_R50_syn0.yaml b/configs/contrastive_ssl/finetune_SSv2_Slow_R50_syn0.yaml similarity index 100% rename from configs/ssl/finetune_SSv2_Slow_R50_syn0.yaml rename to configs/contrastive_ssl/finetune_SSv2_Slow_R50_syn0.yaml diff --git a/configs/ssl/finetune_SSv2_Slow_R50_syn8.yaml b/configs/contrastive_ssl/finetune_SSv2_Slow_R50_syn8.yaml similarity index 100% rename from configs/ssl/finetune_SSv2_Slow_R50_syn8.yaml rename to configs/contrastive_ssl/finetune_SSv2_Slow_R50_syn8.yaml diff --git a/configs/ssl/finetune_ava_Slow_R50_syn0.yaml b/configs/contrastive_ssl/finetune_ava_Slow_R50_syn0.yaml similarity index 100% rename from configs/ssl/finetune_ava_Slow_R50_syn0.yaml rename to configs/contrastive_ssl/finetune_ava_Slow_R50_syn0.yaml diff --git a/configs/ssl/finetune_ava_Slow_R50_syn8.yaml b/configs/contrastive_ssl/finetune_ava_Slow_R50_syn8.yaml similarity index 100% rename from configs/ssl/finetune_ava_Slow_R50_syn8.yaml rename to configs/contrastive_ssl/finetune_ava_Slow_R50_syn8.yaml diff --git a/configs/ssl/finetune_ucf_Slow_R50_syn0.yaml b/configs/contrastive_ssl/finetune_ucf_Slow_R50_syn0.yaml similarity index 100% rename from configs/ssl/finetune_ucf_Slow_R50_syn0.yaml rename to configs/contrastive_ssl/finetune_ucf_Slow_R50_syn0.yaml diff --git a/configs/ssl/finetune_ucf_Slow_R50_syn8.yaml b/configs/contrastive_ssl/finetune_ucf_Slow_R50_syn8.yaml similarity index 100% rename from configs/ssl/finetune_ucf_Slow_R50_syn8.yaml rename to configs/contrastive_ssl/finetune_ucf_Slow_R50_syn8.yaml diff --git a/configs/ssl/linear_k400_Slow_8x8_R50_syn0.yaml b/configs/contrastive_ssl/linear_k400_Slow_8x8_R50_syn0.yaml similarity index 100% rename from configs/ssl/linear_k400_Slow_8x8_R50_syn0.yaml rename to configs/contrastive_ssl/linear_k400_Slow_8x8_R50_syn0.yaml diff --git a/configs/ssl/linear_k400_Slow_8x8_R50_syn8.yaml b/configs/contrastive_ssl/linear_k400_Slow_8x8_R50_syn8.yaml similarity index 100% rename from configs/ssl/linear_k400_Slow_8x8_R50_syn8.yaml rename to configs/contrastive_ssl/linear_k400_Slow_8x8_R50_syn8.yaml diff --git a/configs/masked_ssl/in1k_VIT_B_MaskFeat_FT.yaml b/configs/masked_ssl/in1k_VIT_B_MaskFeat_FT.yaml new file mode 100644 index 000000000..c8f014f75 --- /dev/null +++ b/configs/masked_ssl/in1k_VIT_B_MaskFeat_FT.yaml @@ -0,0 +1,84 @@ +TASK: ssl_eval +TRAIN: + ENABLE: True + DATASET: imagenet + BATCH_SIZE: 512 + EVAL_PERIOD: 10 + CHECKPOINT_PERIOD: 10 + CHECKPOINT_EPOCH_RESET: True + AUTO_RESUME: True +DATA: + MEAN: [0.485, 0.456, 0.406] + STD: [0.229, 0.224, 0.225] + NUM_FRAMES: 1 + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 224 + INPUT_CHANNEL_NUM: [3] +BN: + USE_PRECISE_STATS: True + NUM_BATCHES_PRECISE: 200 +MVIT: + PATCH_2D: True + ZERO_DECAY_POS_CLS: False + MODE: "conv" + CLS_EMBED_ON: True + PATCH_KERNEL: [16, 16] + PATCH_STRIDE: [16, 16] + PATCH_PADDING: [0, 0] + EMBED_DIM: 768 + NUM_HEADS: 12 + MLP_RATIO: 4.0 + QKV_BIAS: True + DROPPATH_RATE: 0.1 + LAYER_SCALE_INIT_VALUE: 0.0 + DEPTH: 12 + NORM: "layernorm" + HEAD_INIT_SCALE: 0.001 + USE_MEAN_POOLING: True +AUG: + ENABLE: True + COLOR_JITTER: 0.4 + AA_TYPE: rand-m9-mstd0.5-inc1 + INTERPOLATION: bicubic + RE_PROB: 0.25 + RE_MODE: pixel + RE_COUNT: 1 + RE_SPLIT: False +MIXUP: + ENABLE: True + ALPHA: 0.8 + CUTMIX_ALPHA: 1.0 + PROB: 1.0 + SWITCH_PROB: 0.5 + LABEL_SMOOTH_VALUE: 0.1 +SOLVER: + BASE_LR_SCALE_NUM_SHARDS: True + LAYER_DECAY: 0.65 + BASE_LR: 0.002 + LR_POLICY: cosine + MAX_EPOCH: 100 + MOMENTUM: 0.9 + WEIGHT_DECAY: 0.05 + WARMUP_EPOCHS: 20.0 + WARMUP_START_LR: 1e-8 + OPTIMIZING_METHOD: adamw + COSINE_AFTER_WARMUP: True + COSINE_END_LR: 1e-6 + ZERO_WD_1D_PARAM: True +MODEL: + NUM_CLASSES: 1000 + ARCH: mvit + MODEL_NAME: MViT + LOSS_FUNC: soft_cross_entropy + DROPOUT_RATE: 0.0 +TEST: + ENABLE: False + DATASET: imagenet + BATCH_SIZE: 256 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git a/configs/masked_ssl/in1k_VIT_B_MaskFeat_PT.yaml b/configs/masked_ssl/in1k_VIT_B_MaskFeat_PT.yaml new file mode 100644 index 000000000..88e0e8997 --- /dev/null +++ b/configs/masked_ssl/in1k_VIT_B_MaskFeat_PT.yaml @@ -0,0 +1,80 @@ +TASK: ssl +TRAIN: + ENABLE: True + DATASET: imagenet + BATCH_SIZE: 256 + EVAL_PERIOD: 10000 + CHECKPOINT_PERIOD: 10 + AUTO_RESUME: True +DATA: + TRAIN_JITTER_SCALES_RELATIVE: [0.5, 1.0] + TRAIN_JITTER_ASPECT_RELATIVE: [0.75, 1.3333] + MEAN: [0.485, 0.456, 0.406] + STD: [0.229, 0.224, 0.225] + NUM_FRAMES: 1 + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 224 + INPUT_CHANNEL_NUM: [3] +BN: + USE_PRECISE_STATS: True + NUM_BATCHES_PRECISE: 200 +MVIT: + PATCH_2D: True + ZERO_DECAY_POS_CLS: False + MODE: "conv" + CLS_EMBED_ON: True + PATCH_KERNEL: [16, 16] + PATCH_STRIDE: [16, 16] + PATCH_PADDING: [0, 0] + EMBED_DIM: 768 + NUM_HEADS: 12 + MLP_RATIO: 4.0 + QKV_BIAS: True + DROPPATH_RATE: 0.0 + DEPTH: 12 + NORM: "layernorm" +MASK: + ENABLE: True + PRETRAIN_DEPTH: [11] + HEAD_TYPE: "separate" + PRED_HOG: True +AUG: + ENABLE: True + RE_PROB: 0.0 + COLOR_JITTER: None + AA_TYPE: "" + GEN_MASK_LOADER: True + MASK_RATIO: 0.4 +MIXUP: + ENABLE: False +SOLVER: + BASE_LR_SCALE_NUM_SHARDS: True + BASE_LR: 0.0002 # bs256 + LR_POLICY: cosine + MAX_EPOCH: 300 + MOMENTUM: 0.9 + WEIGHT_DECAY: 0.05 + WARMUP_EPOCHS: 30.0 + WARMUP_START_LR: 1e-6 + OPTIMIZING_METHOD: adamw + COSINE_AFTER_WARMUP: True + COSINE_END_LR: 1e-6 + ZERO_WD_1D_PARAM: True + CLIP_GRAD_L2NORM: 0.02 +MODEL: + ARCH: maskmvit + MODEL_NAME: MaskMViT + LOSS_FUNC: multi_mse + DROPOUT_RATE: 0.0 +TEST: + ENABLE: False + DATASET: imagenet + BATCH_SIZE: 256 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . +LOG_MODEL_INFO: False diff --git a/configs/masked_ssl/in1k_VIT_L_MaskFeat_FT.yaml b/configs/masked_ssl/in1k_VIT_L_MaskFeat_FT.yaml new file mode 100644 index 000000000..aa356727f --- /dev/null +++ b/configs/masked_ssl/in1k_VIT_L_MaskFeat_FT.yaml @@ -0,0 +1,84 @@ +TRAIN: + ENABLE: True + DATASET: imagenet + BATCH_SIZE: 256 + EVAL_PERIOD: 1 + CHECKPOINT_PERIOD: 5 + CHECKPOINT_EPOCH_RESET: True + AUTO_RESUME: True + MIXED_PRECISION: False +DATA: + MEAN: [0.485, 0.456, 0.406] + STD: [0.229, 0.224, 0.225] + NUM_FRAMES: 1 + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 224 + INPUT_CHANNEL_NUM: [3] +BN: + USE_PRECISE_STATS: True + NUM_BATCHES_PRECISE: 200 +MVIT: + PATCH_2D: True + ZERO_DECAY_POS_CLS: False + MODE: "conv" + CLS_EMBED_ON: True + PATCH_KERNEL: [16, 16] + PATCH_STRIDE: [16, 16] + PATCH_PADDING: [0, 0] + EMBED_DIM: 1024 + NUM_HEADS: 16 + MLP_RATIO: 4.0 + QKV_BIAS: True + DROPPATH_RATE: 0.1 + LAYER_SCALE_INIT_VALUE: 0.0 + DEPTH: 24 + NORM: "layernorm" + HEAD_INIT_SCALE: 0.001 + USE_MEAN_POOLING: True +AUG: + ENABLE: True + COLOR_JITTER: 0.4 + AA_TYPE: rand-m9-mstd0.5-inc1 # ! + INTERPOLATION: bicubic + RE_PROB: 0.25 + RE_MODE: pixel + RE_COUNT: 1 + RE_SPLIT: False +MIXUP: + ENABLE: True + ALPHA: 0.8 + CUTMIX_ALPHA: 1.0 + PROB: 1.0 + SWITCH_PROB: 0.5 + LABEL_SMOOTH_VALUE: 0.1 +SOLVER: + BASE_LR_SCALE_NUM_SHARDS: True + LAYER_DECAY: 0.75 + BASE_LR: 0.001 + LR_POLICY: cosine + MAX_EPOCH: 50 + MOMENTUM: 0.9 + WEIGHT_DECAY: 0.05 + WARMUP_EPOCHS: 5.0 + WARMUP_START_LR: 1e-8 + OPTIMIZING_METHOD: adamw + COSINE_AFTER_WARMUP: True + COSINE_END_LR: 1e-6 + ZERO_WD_1D_PARAM: True +MODEL: + NUM_CLASSES: 1000 + ARCH: mvit + MODEL_NAME: MViT + LOSS_FUNC: soft_cross_entropy + DROPOUT_RATE: 0.0 +TEST: + ENABLE: False + DATASET: imagenet + BATCH_SIZE: 256 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git a/configs/masked_ssl/in1k_VIT_L_MaskFeat_PT.yaml b/configs/masked_ssl/in1k_VIT_L_MaskFeat_PT.yaml new file mode 100644 index 000000000..e3487ebf1 --- /dev/null +++ b/configs/masked_ssl/in1k_VIT_L_MaskFeat_PT.yaml @@ -0,0 +1,80 @@ +TRAIN: + ENABLE: True + DATASET: imagenet + BATCH_SIZE: 256 + EVAL_PERIOD: 10000 + CHECKPOINT_PERIOD: 10 + AUTO_RESUME: True + MIXED_PRECISION: False +DATA: + TRAIN_JITTER_SCALES_RELATIVE: [0.5, 1.0] + TRAIN_JITTER_ASPECT_RELATIVE: [0.75, 1.3333] + MEAN: [0.485, 0.456, 0.406] + STD: [0.229, 0.224, 0.225] + NUM_FRAMES: 1 + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 224 + INPUT_CHANNEL_NUM: [3] +BN: + USE_PRECISE_STATS: True + NUM_BATCHES_PRECISE: 200 +MVIT: + PATCH_2D: True + ZERO_DECAY_POS_CLS: False + MODE: "conv" + CLS_EMBED_ON: True + PATCH_KERNEL: [16, 16] + PATCH_STRIDE: [16, 16] + PATCH_PADDING: [0, 0] + EMBED_DIM: 1024 + NUM_HEADS: 16 + MLP_RATIO: 4.0 + QKV_BIAS: True + DROPPATH_RATE: 0.0 + DEPTH: 24 + NORM: "layernorm" +MASK: + ENABLE: True + PRETRAIN_DEPTH: [23] + HEAD_TYPE: "separate" + PRED_HOG: True +AUG: + ENABLE: True + RE_PROB: 0.0 + COLOR_JITTER: None # important, not using color aug + AA_TYPE: "" # important, not using color aug + GEN_MASK_LOADER: True + MASK_RATIO: 0.4 +MIXUP: + ENABLE: False +SOLVER: + BASE_LR_SCALE_NUM_SHARDS: True + BASE_LR: 0.0002 + LR_POLICY: cosine + MAX_EPOCH: 300 + MOMENTUM: 0.9 + WEIGHT_DECAY: 0.05 + WARMUP_EPOCHS: 30.0 + WARMUP_START_LR: 1e-6 + OPTIMIZING_METHOD: adamw + COSINE_AFTER_WARMUP: True + COSINE_END_LR: 1e-6 + ZERO_WD_1D_PARAM: True + CLIP_GRAD_L2NORM: 0.02 +MODEL: + ARCH: maskmvit + MODEL_NAME: MaskMViT + LOSS_FUNC: multi_mse + DROPOUT_RATE: 0.0 +TEST: + ENABLE: False + DATASET: imagenet + BATCH_SIZE: 256 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . +LOG_MODEL_INFO: False diff --git a/configs/masked_ssl/k400_MVITv2_L_16x4_FT.yaml b/configs/masked_ssl/k400_MVITv2_L_16x4_FT.yaml new file mode 100644 index 000000000..38f9118f2 --- /dev/null +++ b/configs/masked_ssl/k400_MVITv2_L_16x4_FT.yaml @@ -0,0 +1,108 @@ +TASK: ssl_eval +TRAIN: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 16 + EVAL_PERIOD: 5 + CHECKPOINT_PERIOD: 5 + AUTO_RESUME: True + CHECKPOINT_EPOCH_RESET: True +DATA: + USE_OFFSET_SAMPLING: True + DECODING_BACKEND: torchvision + NUM_FRAMES: 16 + SAMPLING_RATE: 4 + TRAIN_JITTER_SCALES: [256, 320] + DECODING_SHORT_SIZE: 320 + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 224 + INPUT_CHANNEL_NUM: [3] + TRAIN_JITTER_SCALES_RELATIVE: [0.08, 1.0] + TRAIN_JITTER_ASPECT_RELATIVE: [0.75, 1.3333] +MVIT: + ZERO_DECAY_POS_CLS: False + SEP_POS_EMBED: True + PATCH_KERNEL: (3, 7, 7) + PATCH_STRIDE: (2, 4, 4) + PATCH_PADDING: (1, 3, 3) + MLP_RATIO: 4.0 + QKV_BIAS: True + NORM: "layernorm" + EMBED_DIM: 144 + NUM_HEADS: 2 + DEPTH: 48 # [2, 6, 36, 2] + DIM_MUL: [[2, 2.0], [8, 2.0], [44, 2.0]] + HEAD_MUL: [[2, 2.0], [8, 2.0], [44, 2.0]] + POOL_Q_STRIDE: [[0, 1, 1, 1], [1, 1, 1, 1], [2, 1, 2, 2], [3, 1, 1, 1], [4, 1, 1, 1], [5, 1, 1, 1], [6, 1, 1, 1], [7, 1, 1, 1], [8, 1, 2, 2], [9, 1, 1, 1], [10, 1, 1, 1], + [11, 1, 1, 1], [12, 1, 1, 1], [13, 1, 1, 1], [14, 1, 1, 1], [15, 1, 1, 1], [16, 1, 1, 1], [17, 1, 1, 1], [18, 1, 1, 1], [19, 1, 1, 1], [20, 1, 1, 1], + [21, 1, 1, 1], [22, 1, 1, 1], [23, 1, 1, 1], [24, 1, 1, 1], [25, 1, 1, 1], [26, 1, 1, 1], [27, 1, 1, 1], [28, 1, 1, 1], [29, 1, 1, 1], [30, 1, 1, 1], + [31, 1, 1, 1], [32, 1, 1, 1], [33, 1, 1, 1], [34, 1, 1, 1], [35, 1, 1, 1], [36, 1, 1, 1], [37, 1, 1, 1], [38, 1, 1, 1], [39, 1, 1, 1], [40, 1, 1, 1], + [41, 1, 1, 1], [42, 1, 1, 1], [43, 1, 1, 1], [44, 1, 2, 2], [45, 1, 1, 1], [46, 1, 1, 1], [47, 1, 1, 1]] + POOL_KV_STRIDE_ADAPTIVE: [1, 8, 8] + POOL_KVQ_KERNEL: [3, 3, 3] + # Please check if your loaded model has cls token. + # It would be better to have cls token. But some of old models forget to turn this on. + CLS_EMBED_ON: True # default: True + # uncomment comment for abs pos: + # SEP_POS_EMBED: True + + # uncomment for rel pos: + USE_ABS_POS: False # default: True + REL_POS_SPATIAL: True # default: false + REL_POS_TEMPORAL: True # default: false + + MODE: "conv" + RESIDUAL_POOLING: True + DROPPATH_RATE: 0.2 + LAYER_SCALE_INIT_VALUE: 0.0 + USE_MEAN_POOLING: True + HEAD_INIT_SCALE: 0.001 +AUG: + ENABLE: True + COLOR_JITTER: None + AA_TYPE: rand-m7-mstd0.5-inc1 + INTERPOLATION: bicubic + RE_PROB: 0.25 + RE_MODE: pixel + RE_COUNT: 1 + RE_SPLIT: False +MIXUP: + ENABLE: True + ALPHA: 0.8 + CUTMIX_ALPHA: 1.0 + PROB: 1.0 + SWITCH_PROB: 0.5 + LABEL_SMOOTH_VALUE: 0.1 +SOLVER: + MAX_EPOCH: 75 + LAYER_DECAY: 0.875 + WARMUP_EPOCHS: 5.0 + CLIP_GRAD_L2NORM: 5.0 + ZERO_WD_1D_PARAM: True + BASE_LR_SCALE_NUM_SHARDS: True + BASE_LR: 0.0006 + COSINE_AFTER_WARMUP: True + COSINE_END_LR: 1e-5 + WARMUP_START_LR: 1e-8 + LR_POLICY: cosine + MOMENTUM: 0.9 + WEIGHT_DECAY: 0.05 + OPTIMIZING_METHOD: adamw +MODEL: + NUM_CLASSES: 400 + ARCH: mvit + MODEL_NAME: MViT + LOSS_FUNC: soft_cross_entropy # default cross_entropy + DROPOUT_RATE: 0.0 +TEST: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 64 + NUM_SPATIAL_CROPS: 1 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git a/configs/masked_ssl/k400_MVITv2_L_16x4_MaskFeat_PT.yaml b/configs/masked_ssl/k400_MVITv2_L_16x4_MaskFeat_PT.yaml new file mode 100644 index 000000000..0ce804617 --- /dev/null +++ b/configs/masked_ssl/k400_MVITv2_L_16x4_MaskFeat_PT.yaml @@ -0,0 +1,105 @@ +TASK: ssl +TRAIN: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 32 + EVAL_PERIOD: 100000 + CHECKPOINT_PERIOD: 5 + AUTO_RESUME: True +DATA: + USE_OFFSET_SAMPLING: True + DECODING_BACKEND: torchvision + NUM_FRAMES: 16 + SAMPLING_RATE: 4 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 224 + INPUT_CHANNEL_NUM: [3] + DECODING_SHORT_SIZE: 320 + TRAIN_JITTER_SCALES_RELATIVE: [0.5, 1.0] + TRAIN_JITTER_ASPECT_RELATIVE: [0.75, 1.3333] +MVIT: + ZERO_DECAY_POS_CLS: False + SEP_POS_EMBED: True + PATCH_KERNEL: (3, 7, 7) + PATCH_STRIDE: (2, 4, 4) + PATCH_PADDING: (1, 3, 3) + MLP_RATIO: 4.0 + QKV_BIAS: True + NORM: "layernorm" + EMBED_DIM: 144 + NUM_HEADS: 2 + DEPTH: 48 + DIM_MUL: [[2, 2.0], [8, 2.0], [44, 2.0]] + HEAD_MUL: [[2, 2.0], [8, 2.0], [44, 2.0]] + # Highlight: [44, 1, 1, 1] instead of [44, 1, 2, 2] for 14x14 output + POOL_Q_STRIDE: [[0, 1, 1, 1], [1, 1, 1, 1], [2, 1, 2, 2], [3, 1, 1, 1], [4, 1, 1, 1], [5, 1, 1, 1], [6, 1, 1, 1], [7, 1, 1, 1], [8, 1, 2, 2], [9, 1, 1, 1], [10, 1, 1, 1], + [11, 1, 1, 1], [12, 1, 1, 1], [13, 1, 1, 1], [14, 1, 1, 1], [15, 1, 1, 1], [16, 1, 1, 1], [17, 1, 1, 1], [18, 1, 1, 1], [19, 1, 1, 1], [20, 1, 1, 1], + [21, 1, 1, 1], [22, 1, 1, 1], [23, 1, 1, 1], [24, 1, 1, 1], [25, 1, 1, 1], [26, 1, 1, 1], [27, 1, 1, 1], [28, 1, 1, 1], [29, 1, 1, 1], [30, 1, 1, 1], + [31, 1, 1, 1], [32, 1, 1, 1], [33, 1, 1, 1], [34, 1, 1, 1], [35, 1, 1, 1], [36, 1, 1, 1], [37, 1, 1, 1], [38, 1, 1, 1], [39, 1, 1, 1], [40, 1, 1, 1], + [41, 1, 1, 1], [42, 1, 1, 1], [43, 1, 1, 1], [44, 1, 1, 1], [45, 1, 1, 1], [46, 1, 1, 1], [47, 1, 1, 1]] + DROPPATH_RATE: 0.0 + POOL_KV_STRIDE_ADAPTIVE: [1, 8, 8] + POOL_KVQ_KERNEL: [3, 3, 3] + CLS_EMBED_ON: True # defauult: True + # uncomment comment for abs pos: + # SEP_POS_EMBED: True + + # uncomment for rel pos: + USE_ABS_POS: False # default: True + REL_POS_SPATIAL: True # default: false + REL_POS_TEMPORAL: True # default: false + + MODE: "conv" + RESIDUAL_POOLING: True +MASK: + ENABLE: True + PRETRAIN_DEPTH: [47] + PRED_HOG: True # default: false +AUG: + ENABLE: True + COLOR_JITTER: None + AA_TYPE: "" + RE_PROB: 0.0 + GEN_MASK_LOADER: True + MASK_RATIO: 0.4 + + # Mask Cube (Default) + MASK_TUBE: False + MASK_FRAMES: False + MASK_WINDOW_SIZE: [8, 7, 7] + +MIXUP: + ENABLE: False +SOLVER: + CLIP_GRAD_L2NORM: 0.02 + BASE_LR: 0.0001 + BASE_LR_SCALE_NUM_SHARDS: True + LR_POLICY: cosine + COSINE_AFTER_WARMUP: True + COSINE_END_LR: 1e-6 + MAX_EPOCH: 300 + WARMUP_EPOCHS: 10.0 + MOMENTUM: 0.9 + WEIGHT_DECAY: 0.05 + ZERO_WD_1D_PARAM: True + WARMUP_START_LR: 1e-6 + OPTIMIZING_METHOD: adamw +MODEL: + ARCH: maskmvit + MODEL_NAME: MaskMViT + LOSS_FUNC: multi_mse + DROPOUT_RATE: 0.0 +TEST: + ENABLE: False + DATASET: kinetics + BATCH_SIZE: 64 + NUM_SPATIAL_CROPS: 1 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . +LOG_MODEL_INFO: False diff --git a/configs/masked_ssl/k400_MVITv2_S_16x4_FT.yaml b/configs/masked_ssl/k400_MVITv2_S_16x4_FT.yaml new file mode 100644 index 000000000..45460b9c7 --- /dev/null +++ b/configs/masked_ssl/k400_MVITv2_S_16x4_FT.yaml @@ -0,0 +1,101 @@ +TASK: ssl_eval +TRAIN: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 16 + EVAL_PERIOD: 10 + CHECKPOINT_EPOCH_RESET: True + CHECKPOINT_PERIOD: 10 + AUTO_RESUME: True + MIXED_PRECISION: False +DATA: + USE_OFFSET_SAMPLING: True + DECODING_BACKEND: torchvision + NUM_FRAMES: 16 + SAMPLING_RATE: 4 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 224 + INPUT_CHANNEL_NUM: [3] + TRAIN_JITTER_SCALES_RELATIVE: [0.08, 1.0] + TRAIN_JITTER_ASPECT_RELATIVE: [0.75, 1.3333] + DECODING_SHORT_SIZE: 320 +MVIT: + DEPTH: 16 # def 12 + NUM_HEADS: 3 # default 12 16 + EMBED_DIM: 384 # def 192 768 384 1024 + NUM_HEADS: 1 # default 12 16 + EMBED_DIM: 96 # def 256 192 768 384 1024 + PATCH_KERNEL: [3, 7, 7] + PATCH_STRIDE: [2, 4, 4] # default:empty + PATCH_PADDING: [1, 3, 3] + ZERO_DECAY_POS_CLS: False + QKV_BIAS: True + DIM_MUL: [[1, 2.0], [3, 2.0], [14, 2.0]] + HEAD_MUL: [[1, 2.0], [3, 2.0], [14, 2.0]] + POOL_Q_STRIDE: [[0, 1, 1, 1], [1, 1, 2, 2], [2, 1, 1, 1], [3, 1, 2, 2], [4, 1, 1, 1], [5, 1, 1, 1], [6, 1, 1, 1], [7, 1, 1, 1], [8, 1, 1, 1], [9, 1, 1, 1], [10, 1, 1, 1], [11, 1, 1, 1], [12, 1, 1, 1], [13, 1, 1, 1], [14, 1, 2, 2], [15, 1, 1, 1]] + POOL_KVQ_KERNEL: [3, 3, 3] + POOL_KV_STRIDE_ADAPTIVE: [1, 8, 8] + CLS_EMBED_ON: True # default: True + USE_ABS_POS: False # default: True + SEP_POS_EMBED: True # default: false + REL_POS_SPATIAL: True # default: false + REL_POS_TEMPORAL: True # default: false + RESIDUAL_POOLING: True + MODE: "conv" + DROPPATH_RATE: 0.1 + LAYER_SCALE_INIT_VALUE: 0.0 + USE_MEAN_POOLING: True + HEAD_INIT_SCALE: 0.001 +AUG: + ENABLE: True + COLOR_JITTER: 0.4 + AA_TYPE: rand-m7-mstd0.5-inc1 + INTERPOLATION: bicubic + NUM_SAMPLE: 2 + RE_PROB: 0.25 + RE_MODE: pixel + RE_COUNT: 1 + RE_SPLIT: False +MIXUP: + ENABLE: True + ALPHA: 0.8 + CUTMIX_ALPHA: 1.0 + PROB: 1.0 + SWITCH_PROB: 0.5 + LABEL_SMOOTH_VALUE: 0.1 +SOLVER: + ZERO_WD_1D_PARAM: True + CLIP_GRAD_L2NORM: 5.0 + LAYER_DECAY: 0.75 # ! + BASE_LR_SCALE_NUM_SHARDS: True + BASE_LR: 0.0006 # ! + COSINE_AFTER_WARMUP: True + COSINE_END_LR: 1e-6 + WARMUP_START_LR: 1e-6 + WARMUP_EPOCHS: 20.0 + LR_POLICY: cosine + MAX_EPOCH: 100 + MOMENTUM: 0.9 + WEIGHT_DECAY: 0.05 + OPTIMIZING_METHOD: adamw + COSINE_AFTER_WARMUP: True +MODEL: + NUM_CLASSES: 400 + ARCH: mvit + MODEL_NAME: MViT + LOSS_FUNC: soft_cross_entropy + DROPOUT_RATE: 0.0 +TEST: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 64 + NUM_SPATIAL_CROPS: 1 + NUM_TEMPORAL_CLIPS: [1, 3, 4, 5, 7, 10] +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git a/configs/masked_ssl/k400_MVITv2_S_16x4_MaskFeat_PT.yaml b/configs/masked_ssl/k400_MVITv2_S_16x4_MaskFeat_PT.yaml new file mode 100644 index 000000000..99388faf8 --- /dev/null +++ b/configs/masked_ssl/k400_MVITv2_S_16x4_MaskFeat_PT.yaml @@ -0,0 +1,98 @@ +TASK: ssl +TRAIN: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 32 + EVAL_PERIOD: 1000000 + CHECKPOINT_PERIOD: 10 + AUTO_RESUME: True +DATA: + USE_OFFSET_SAMPLING: True + DECODING_BACKEND: torchvision + NUM_FRAMES: 16 + SAMPLING_RATE: 4 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 224 + INPUT_CHANNEL_NUM: [3] + DECODING_SHORT_SIZE: 320 + TRAIN_JITTER_SCALES_RELATIVE: [0.5, 1.0] + TRAIN_JITTER_ASPECT_RELATIVE: [0.75, 1.3333] +MVIT: + DEPTH: 16 + NUM_HEADS: 1 + EMBED_DIM: 96 + PATCH_KERNEL: [3, 7, 7] + PATCH_STRIDE: [2, 4, 4] + PATCH_PADDING: [1, 3, 3] + ZERO_DECAY_POS_CLS: False + QKV_BIAS: True + DIM_MUL: [[1, 2.0], [3, 2.0], [14, 2.0]] + HEAD_MUL: [[1, 2.0], [3, 2.0], [14, 2.0]] + # Highlight: [14, 1, 1, 1] instead of [14, 1, 2, 2] for 14x14 output + POOL_Q_STRIDE: [[0, 1, 1, 1], [1, 1, 2, 2], [2, 1, 1, 1], [3, 1, 2, 2], [4, 1, 1, 1], [5, 1, 1, 1], [6, 1, 1, 1], [7, 1, 1, 1], [8, 1, 1, 1], [9, 1, 1, 1], [10, 1, 1, 1], [11, 1, 1, 1], [12, 1, 1, 1], [13, 1, 1, 1], [14, 1, 1, 1], [15, 1, 1, 1]] + POOL_KVQ_KERNEL: [3, 3, 3] + POOL_KV_STRIDE_ADAPTIVE: [1, 8, 8] + DROPPATH_RATE: 0.0 + CLS_EMBED_ON: True # default: True + USE_ABS_POS: False # default: True + SEP_POS_EMBED: True # default: false + REL_POS_SPATIAL: True # default: false + REL_POS_TEMPORAL: True # default: false + RESIDUAL_POOLING: True + MODE: "conv" +MASK: + ENABLE: True + PRETRAIN_DEPTH: [15] + HEAD_TYPE: "separate" + PRED_HOG: True +AUG: + ENABLE: True + COLOR_JITTER: None + AA_TYPE: "" + RE_PROB: 0.0 + GEN_MASK_LOADER: True + MASK_RATIO: 0.4 + + # Mask Cube (Default) + MASK_TUBE: False + MASK_FRAMES: False + MASK_WINDOW_SIZE: [8, 7, 7] +MIXUP: + ENABLE: False +BN: + USE_PRECISE_STATS: False + NUM_BATCHES_PRECISE: 200 +SOLVER: + CLIP_GRAD_L2NORM: 0.02 + BASE_LR: 0.0001 + BASE_LR_SCALE_NUM_SHARDS: True + LR_POLICY: cosine + COSINE_AFTER_WARMUP: True + COSINE_END_LR: 1e-6 + MAX_EPOCH: 300 + WARMUP_EPOCHS: 10.0 + MOMENTUM: 0.9 + WEIGHT_DECAY: 0.05 + ZERO_WD_1D_PARAM: True + WARMUP_START_LR: 1e-6 + OPTIMIZING_METHOD: adamw +MODEL: + ARCH: maskmvit + MODEL_NAME: MaskMViT + LOSS_FUNC: multi_mse + DROPOUT_RATE: 0.0 +TEST: + ENABLE: False + DATASET: kinetics + BATCH_SIZE: 16 + NUM_SPATIAL_CROPS: 1 + NUM_TEMPORAL_CLIPS: [5, 10] +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . +LOG_MODEL_INFO: False diff --git a/configs/masked_ssl/k400_VIT_B_16x4_FT.yaml b/configs/masked_ssl/k400_VIT_B_16x4_FT.yaml new file mode 100644 index 000000000..ccce0f7e7 --- /dev/null +++ b/configs/masked_ssl/k400_VIT_B_16x4_FT.yaml @@ -0,0 +1,90 @@ +TASK: ssl_eval +TRAIN: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 1 + EVAL_PERIOD: 1 + CHECKPOINT_PERIOD: 5 + AUTO_RESUME: True + CHECKPOINT_EPOCH_RESET: True +DATA: + USE_OFFSET_SAMPLING: True + DECODING_BACKEND: torchvision + NUM_FRAMES: 16 + SAMPLING_RATE: 4 + TRAIN_JITTER_SCALES: [256, 320] + DECODING_SHORT_SIZE: 320 + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 224 + INPUT_CHANNEL_NUM: [3] + TRAIN_JITTER_SCALES_RELATIVE: [0.08, 1.0] + TRAIN_JITTER_ASPECT_RELATIVE: [0.75, 1.3333] +MVIT: + ZERO_DECAY_POS_CLS: False + SEP_POS_EMBED: True + PATCH_KERNEL: (2, 16, 16) + PATCH_STRIDE: (2, 16, 16) + PATCH_PADDING: (0, 0, 0) + EMBED_DIM: 768 # vit-B, change depth & layerdecay + NUM_HEADS: 12 + MLP_RATIO: 4.0 + QKV_BIAS: True + NORM: "layernorm" + DEPTH: 12 + MODE: "conv" + DROPPATH_RATE: 0.1 + LAYER_SCALE_INIT_VALUE: 0.0 + USE_MEAN_POOLING: True + HEAD_INIT_SCALE: 0.001 +AUG: + NUM_SAMPLE: 2 + ENABLE: True + COLOR_JITTER: None + AA_TYPE: rand-m7-mstd0.5-inc1 + INTERPOLATION: bicubic + RE_PROB: 0.25 + RE_MODE: pixel + RE_COUNT: 1 + RE_SPLIT: False +MIXUP: + ENABLE: True + ALPHA: 0.8 + CUTMIX_ALPHA: 1.0 + PROB: 1.0 + SWITCH_PROB: 0.5 + LABEL_SMOOTH_VALUE: 0.1 +SOLVER: + MAX_EPOCH: 75 + LAYER_DECAY: 0.75 + LAYER_DECAY: 0.65 # vit-B + WARMUP_EPOCHS: 5.0 + CLIP_GRAD_L2NORM: 5.0 + ZERO_WD_1D_PARAM: True + BASE_LR_SCALE_NUM_SHARDS: True + BASE_LR: 6e-4 + COSINE_AFTER_WARMUP: True + COSINE_END_LR: 1e-5 + WARMUP_START_LR: 1e-8 + LR_POLICY: cosine + MOMENTUM: 0.9 + WEIGHT_DECAY: 0.05 + OPTIMIZING_METHOD: adamw +MODEL: + NUM_CLASSES: 400 + ARCH: mvit + MODEL_NAME: MViT + LOSS_FUNC: soft_cross_entropy # default cross_entropy + DROPOUT_RATE: 0.3 +TEST: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 64 + NUM_SPATIAL_CROPS: 3 + NUM_TEMPORAL_CLIPS: [5, 7, 10] +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git a/configs/masked_ssl/k400_VIT_B_16x4_MAE_PT.yaml b/configs/masked_ssl/k400_VIT_B_16x4_MAE_PT.yaml new file mode 100644 index 000000000..5114d754d --- /dev/null +++ b/configs/masked_ssl/k400_VIT_B_16x4_MAE_PT.yaml @@ -0,0 +1,86 @@ +TASK: ssl +TRAIN: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 64 + EVAL_PERIOD: 100000 + CHECKPOINT_PERIOD: 5 + AUTO_RESUME: True + KILL_LOSS_EXPLOSION_FACTOR: 2.0 # default 0.0 (not enforced) +DATA: + USE_OFFSET_SAMPLING: True + DECODING_BACKEND: torchvision + NUM_FRAMES: 16 + SAMPLING_RATE: 4 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 224 + INPUT_CHANNEL_NUM: [3] + DECODING_SHORT_SIZE: 320 + TRAIN_JITTER_SCALES_RELATIVE: [0.5, 1.0] + TRAIN_JITTER_ASPECT_RELATIVE: [0.75, 1.3333] + TRAIN_CROP_NUM_TEMPORAL: 4 # default 1 +MVIT: + ZERO_DECAY_POS_CLS: False + SEP_POS_EMBED: True # default: False needs to be false for fixed sincos + PATCH_KERNEL: (2, 16, 16) + PATCH_STRIDE: (2, 16, 16) + PATCH_PADDING: (0, 0, 0) + EMBED_DIM: 768 # vit-B + NUM_HEADS: 12 + MLP_RATIO: 4.0 + QKV_BIAS: True + NORM: "layernorm" + DEPTH: 12 + DROPPATH_RATE: 0.0 + MODE: "conv" + CLS_EMBED_ON: True +MASK: + ENABLE: True + MAE_ON: True # default False + MAE_RND_MASK: True # default: false + PRETRAIN_DEPTH: [11] + HEAD_TYPE: "separate_xformer" + DECODER_DEPTH: 4 + DECODER_EMBED_DIM: 512 +AUG: + ENABLE: True + COLOR_JITTER: None + AA_TYPE: "" + RE_PROB: 0.0 + MASK_RATIO: 0.9 +MIXUP: + ENABLE: False +SOLVER: + CLIP_GRAD_L2NORM: 0.02 + BASE_LR: 1e-4 + BASE_LR_SCALE_NUM_SHARDS: True + LR_POLICY: cosine + COSINE_AFTER_WARMUP: True + COSINE_END_LR: 1e-6 + MAX_EPOCH: 200 + WARMUP_EPOCHS: 60.0 + BETAS: (0.9, 0.95) # default (0.9, 0.999) + MOMENTUM: 0.9 + WEIGHT_DECAY: 0.05 + ZERO_WD_1D_PARAM: True # default False + WARMUP_START_LR: 1e-6 + OPTIMIZING_METHOD: adamw +MODEL: + ARCH: maskmvit + MODEL_NAME: MaskMViT + LOSS_FUNC: multi_mse + DROPOUT_RATE: 0.0 +TEST: + ENABLE: False + DATASET: kinetics + BATCH_SIZE: 64 + NUM_SPATIAL_CROPS: 1 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . +LOG_MODEL_INFO: False diff --git a/configs/masked_ssl/k400_VIT_H_16x4_FT.yaml b/configs/masked_ssl/k400_VIT_H_16x4_FT.yaml new file mode 100644 index 000000000..bacda1650 --- /dev/null +++ b/configs/masked_ssl/k400_VIT_H_16x4_FT.yaml @@ -0,0 +1,88 @@ +TASK: ssl_eval +TRAIN: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 16 + EVAL_PERIOD: 5 + CHECKPOINT_PERIOD: 5 + AUTO_RESUME: True + CHECKPOINT_EPOCH_RESET: True +DATA: + USE_OFFSET_SAMPLING: True + DECODING_BACKEND: torchvision + NUM_FRAMES: 16 + SAMPLING_RATE: 4 + TRAIN_JITTER_SCALES: [256, 320] + DECODING_SHORT_SIZE: 320 + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 224 + INPUT_CHANNEL_NUM: [3] + TRAIN_JITTER_SCALES_RELATIVE: [0.08, 1.0] + TRAIN_JITTER_ASPECT_RELATIVE: [0.75, 1.3333] +MVIT: + ZERO_DECAY_POS_CLS: False + SEP_POS_EMBED: True + PATCH_KERNEL: (2, 16, 16) + PATCH_STRIDE: (2, 16, 16) + PATCH_PADDING: (0, 0, 0) + EMBED_DIM: 1280 + NUM_HEADS: 16 + MLP_RATIO: 4.0 + QKV_BIAS: True + NORM: "layernorm" + DEPTH: 32 + MODE: "conv" + DROPPATH_RATE: 0.2 + LAYER_SCALE_INIT_VALUE: 0.0 + USE_MEAN_POOLING: True + HEAD_INIT_SCALE: 0.001 +AUG: + ENABLE: True + COLOR_JITTER: None + AA_TYPE: rand-m7-mstd0.5-inc1 + INTERPOLATION: bicubic + RE_PROB: 0.25 + RE_MODE: pixel + RE_COUNT: 1 + RE_SPLIT: False +MIXUP: + ENABLE: True + ALPHA: 0.8 + CUTMIX_ALPHA: 1.0 + PROB: 1.0 + SWITCH_PROB: 0.5 + LABEL_SMOOTH_VALUE: 0.1 +SOLVER: + MAX_EPOCH: 75 + LAYER_DECAY: 0.8 + WARMUP_EPOCHS: 5.0 + CLIP_GRAD_L2NORM: 5.0 + ZERO_WD_1D_PARAM: True + BASE_LR_SCALE_NUM_SHARDS: True + BASE_LR: 3e-4 # works best + COSINE_AFTER_WARMUP: True + COSINE_END_LR: 1e-6 + WARMUP_START_LR: 1e-8 + LR_POLICY: cosine + MOMENTUM: 0.9 + WEIGHT_DECAY: 0.05 + OPTIMIZING_METHOD: adamw +MODEL: + NUM_CLASSES: 400 + ARCH: mvit + MODEL_NAME: MViT + LOSS_FUNC: soft_cross_entropy # default cross_entropy + DROPOUT_RATE: 0.5 +TEST: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 64 + NUM_SPATIAL_CROPS: 3 + NUM_TEMPORAL_CLIPS: [5, 7, 10] +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git a/configs/masked_ssl/k400_VIT_H_16x4_MAE_PT.yaml b/configs/masked_ssl/k400_VIT_H_16x4_MAE_PT.yaml new file mode 100644 index 000000000..5e4504612 --- /dev/null +++ b/configs/masked_ssl/k400_VIT_H_16x4_MAE_PT.yaml @@ -0,0 +1,88 @@ +TASK: ssl +TRAIN: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 32 + EVAL_PERIOD: 100000 + CHECKPOINT_PERIOD: 5 + AUTO_RESUME: True + KILL_LOSS_EXPLOSION_FACTOR: 2.0 # default 0.0 (not enforced) +DATA: + USE_OFFSET_SAMPLING: True + DECODING_BACKEND: torchvision + NUM_FRAMES: 16 + SAMPLING_RATE: 4 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 224 + INPUT_CHANNEL_NUM: [3] + DECODING_SHORT_SIZE: 320 + TRAIN_JITTER_SCALES_RELATIVE: [0.5, 1.0] + TRAIN_JITTER_ASPECT_RELATIVE: [0.75, 1.3333] + TRAIN_CROP_NUM_TEMPORAL: 4 # default 1 +MVIT: + ZERO_DECAY_POS_CLS: False + SEP_POS_EMBED: True + PATCH_KERNEL: (2, 16, 16) + PATCH_STRIDE: (2, 16, 16) + PATCH_PADDING: (0, 0, 0) + EMBED_DIM: 1280 + NUM_HEADS: 16 + MLP_RATIO: 4.0 + QKV_BIAS: True + NORM: "layernorm" + DEPTH: 32 + DROPPATH_RATE: 0.0 + MODE: "conv" + CLS_EMBED_ON: True # default: True +MASK: + ENABLE: True + MAE_ON: True # default False + MAE_RND_MASK: True # default: false + PRETRAIN_DEPTH: [31] + HEAD_TYPE: "separate_xformer" + DECODER_DEPTH: 4 # default: 0 + DECODER_EMBED_DIM: 512 +AUG: + NUM_SAMPLE: 1 + ENABLE: True + COLOR_JITTER: None + AA_TYPE: "" + RE_PROB: 0.0 + MASK_RATIO: 0.9 + +MIXUP: + ENABLE: False +SOLVER: + CLIP_GRAD_L2NORM: 0.02 + BASE_LR: 1e-4 # for bs32 + BASE_LR_SCALE_NUM_SHARDS: True + LR_POLICY: cosine + COSINE_AFTER_WARMUP: True + COSINE_END_LR: 1e-6 + MAX_EPOCH: 400 + WARMUP_EPOCHS: 60.0 + BETAS: (0.9, 0.95) # default (0.9, 0.999) + MOMENTUM: 0.9 + WEIGHT_DECAY: 0.05 + ZERO_WD_1D_PARAM: True # default False + WARMUP_START_LR: 1e-6 + OPTIMIZING_METHOD: adamw +MODEL: + ARCH: maskmvit + MODEL_NAME: MaskMViT + LOSS_FUNC: multi_mse + DROPOUT_RATE: 0.0 +TEST: + ENABLE: False + DATASET: kinetics + BATCH_SIZE: 64 + NUM_SPATIAL_CROPS: 1 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . +LOG_MODEL_INFO: False diff --git a/configs/masked_ssl/k400_VIT_L_16x4_FT.yaml b/configs/masked_ssl/k400_VIT_L_16x4_FT.yaml new file mode 100644 index 000000000..68fa2d0ed --- /dev/null +++ b/configs/masked_ssl/k400_VIT_L_16x4_FT.yaml @@ -0,0 +1,89 @@ +TASK: ssl_eval +TRAIN: + DATASET: kinetics + BATCH_SIZE: 16 + EVAL_PERIOD: 5 + CHECKPOINT_PERIOD: 5 + AUTO_RESUME: True + CHECKPOINT_EPOCH_RESET: True +DATA: + USE_OFFSET_SAMPLING: True + DECODING_BACKEND: torchvision + NUM_FRAMES: 16 + SAMPLING_RATE: 4 + TRAIN_JITTER_SCALES: [256, 320] + DECODING_SHORT_SIZE: 320 + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 224 + INPUT_CHANNEL_NUM: [3] + TRAIN_JITTER_SCALES_RELATIVE: [0.08, 1.0] + TRAIN_JITTER_ASPECT_RELATIVE: [0.75, 1.3333] +MVIT: + ZERO_DECAY_POS_CLS: False + SEP_POS_EMBED: True + PATCH_KERNEL: (2, 16, 16) + PATCH_STRIDE: (2, 16, 16) + PATCH_PADDING: (0, 0, 0) + EMBED_DIM: 1024 + NUM_HEADS: 16 + MLP_RATIO: 4.0 + QKV_BIAS: True + NORM: "layernorm" + DEPTH: 24 + MODE: "conv" + DROPPATH_RATE: 0.2 + LAYER_SCALE_INIT_VALUE: 0.0 + USE_MEAN_POOLING: True + HEAD_INIT_SCALE: 0.001 +AUG: + NUM_SAMPLE: 2 + ENABLE: True + COLOR_JITTER: None + AA_TYPE: rand-m7-mstd0.5-inc1 + INTERPOLATION: bicubic + RE_PROB: 0.25 + RE_MODE: pixel + RE_COUNT: 1 + RE_SPLIT: False +MIXUP: + ENABLE: True + ALPHA: 0.8 + CUTMIX_ALPHA: 1.0 + PROB: 1.0 + SWITCH_PROB: 0.5 + LABEL_SMOOTH_VALUE: 0.1 +SOLVER: + MAX_EPOCH: 75 + MAX_EPOCH: 50 + LAYER_DECAY: 0.75 + WARMUP_EPOCHS: 5.0 + CLIP_GRAD_L2NORM: 5.0 + ZERO_WD_1D_PARAM: True + BASE_LR_SCALE_NUM_SHARDS: True + BASE_LR: 3e-4 # works best + COSINE_AFTER_WARMUP: True + COSINE_END_LR: 1e-5 + WARMUP_START_LR: 1e-8 + LR_POLICY: cosine + MOMENTUM: 0.9 + WEIGHT_DECAY: 0.05 + OPTIMIZING_METHOD: adamw +MODEL: + NUM_CLASSES: 400 + ARCH: mvit + MODEL_NAME: MViT + LOSS_FUNC: soft_cross_entropy # default cross_entropy + DROPOUT_RATE: 0.3 +TEST: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 64 + NUM_SPATIAL_CROPS: 3 + NUM_TEMPORAL_CLIPS: [5, 7, 10] +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git a/configs/masked_ssl/k400_VIT_L_16x4_MAE_PT.yaml b/configs/masked_ssl/k400_VIT_L_16x4_MAE_PT.yaml new file mode 100644 index 000000000..51bf33f07 --- /dev/null +++ b/configs/masked_ssl/k400_VIT_L_16x4_MAE_PT.yaml @@ -0,0 +1,87 @@ +TASK: ssl +TRAIN: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 32 # 32 fits 32g + EVAL_PERIOD: 100000 + CHECKPOINT_PERIOD: 5 + AUTO_RESUME: True + KILL_LOSS_EXPLOSION_FACTOR: 2.0 # default 0.0 (not enforced) +DATA: + USE_OFFSET_SAMPLING: True + DECODING_BACKEND: torchvision + NUM_FRAMES: 16 + SAMPLING_RATE: 4 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 224 + INPUT_CHANNEL_NUM: [3] + DECODING_SHORT_SIZE: 320 + TRAIN_JITTER_SCALES_RELATIVE: [0.5, 1.0] + TRAIN_JITTER_ASPECT_RELATIVE: [0.75, 1.3333] + TRAIN_CROP_NUM_TEMPORAL: 4 # default 1 +MVIT: + ZERO_DECAY_POS_CLS: False + SEP_POS_EMBED: True # default: False needs to be false for fixed sincos + PATCH_KERNEL: (2, 16, 16) + PATCH_STRIDE: (2, 16, 16) + PATCH_PADDING: (0, 0, 0) + EMBED_DIM: 1024 + NUM_HEADS: 16 + MLP_RATIO: 4.0 + QKV_BIAS: True + NORM: "layernorm" + DEPTH: 24 + DROPPATH_RATE: 0.0 + MODE: "conv" + CLS_EMBED_ON: True # default: True +MASK: + ENABLE: True + MAE_ON: True # default False + MAE_RND_MASK: True # default: false + PRETRAIN_DEPTH: [23] + HEAD_TYPE: "separate_xformer" + DECODER_DEPTH: 4 # default: 0 + DECODER_EMBED_DIM: 512 +AUG: + # NUM_SAMPLE: 4 + ENABLE: True + COLOR_JITTER: None + AA_TYPE: "" + RE_PROB: 0.0 + MASK_RATIO: 0.9 +MIXUP: + ENABLE: False +SOLVER: + CLIP_GRAD_L2NORM: 0.02 + BASE_LR: 1e-4 # for bs64 + BASE_LR_SCALE_NUM_SHARDS: True + LR_POLICY: cosine + COSINE_AFTER_WARMUP: True + COSINE_END_LR: 1e-6 + MAX_EPOCH: 200 + WARMUP_EPOCHS: 30.0 + BETAS: (0.9, 0.95) # default (0.9, 0.999) + MOMENTUM: 0.9 + WEIGHT_DECAY: 0.05 + ZERO_WD_1D_PARAM: True # default False + WARMUP_START_LR: 1e-6 + OPTIMIZING_METHOD: adamw +MODEL: + ARCH: maskmvit + MODEL_NAME: MaskMViT + LOSS_FUNC: multi_mse + DROPOUT_RATE: 0.0 +TEST: + ENABLE: False + DATASET: kinetics + BATCH_SIZE: 64 + NUM_SPATIAL_CROPS: 1 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . +LOG_MODEL_INFO: False diff --git a/projects/ssl/README.md b/projects/contrastive_ssl/README.md similarity index 73% rename from projects/ssl/README.md rename to projects/contrastive_ssl/README.md index 4ffae6586..231b44cab 100644 --- a/projects/ssl/README.md +++ b/projects/contrastive_ssl/README.md @@ -13,21 +13,21 @@ In CVPR, 2021. [[Paper](https://arxiv.org/abs/2104.14558)] | method | ρ| architecture | size | frames x sampling | pretrain data | K400-linear | UCF101-split1 | AVA | SSv2 | model | config | | ------------- | ------------- | ------------- | ------------- | ------------- | ------------- | ------------- | ------------- | ------------- | ------------- | ------------- | ------------- | -| MoCo | 2 | Slow-only | R50 | 8 x 8 | Kinetics-400 | 66.6 | 91.3 | 19.7 | 52.7 | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/videomoco_models/MoCo_SlowR50_8x8_T2_epoch_00200.pyth) | ssl/MoCo_SlowR50_8x8 | -| BYOL | 2 | Slow-only | R50 | 8 x 8 | Kinetics-400 | 67.4 | 94.0 | 22.8 | 54.4 | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/videomoco_models/BYOL_SlowR50_8x8_T2_epoch_00200.pyth) | ssl/BYOL_SlowR50_8x8 | -| SimCLR | 2 | Slow-only | R50 | 8 x 8 | Kinetics-400 | 61.5 | 88.3 | 17.5 | 51.4 | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/videomoco_models/SimCLR_SlowR50_8x8_T2_epoch_00200.pyth) | ssl/SimCLR_SlowR50_8x8 | -| SwAV | 2 | Slow-only | R50 | 8 x 8 | Kinetics-400 | 62.6 | 90.2 | 19.2 | 52.5| [`link`](https://dl.fbaipublicfiles.com/pyslowfast/videomoco_models/SwAV_SlowR50_8x8_T2_epoch_00200.pyth) | ssl/SwAV_SlowR50_8x8 | -| MoCo | 4 | Slow-only | R50 | 8 x 8 | Kinetics-400 | 71.0 | 94.5 | 21.9 | 54.0 | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/videomoco_models/MoCo_SlowR50_8x8_T4_epoch_00200.pyth) | ssl/MoCo_SlowR50_8x8 | -| BYOL | 4 | Slow-only | R50 | 8 x 8 | Kinetics-400 | 70.1 | 94.7 | xx | xx | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/videomoco_models/BYOL_SlowR50_8x8_T4_epoch_00200.pyth) | ssl/BYOL_SlowR50_8x8 | -| BYOL | 4 | Slow-only | R50 | 16 x 4 | Kinetics-400 | 71.1 | 95.4 | xx | xx | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/videomoco_models/BYOL_SlowR50_16x4_T4_epoch_00200.pyth) | ssl/BYOL_SlowR50_8x8 | +| MoCo | 2 | Slow-only | R50 | 8 x 8 | Kinetics-400 | 66.6 | 91.3 | 19.7 | 52.7 | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/videomoco_models/MoCo_SlowR50_8x8_T2_epoch_00200.pyth) | contrastive_ssl/MoCo_SlowR50_8x8 | +| BYOL | 2 | Slow-only | R50 | 8 x 8 | Kinetics-400 | 67.4 | 94.0 | 22.8 | 54.4 | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/videomoco_models/BYOL_SlowR50_8x8_T2_epoch_00200.pyth) | contrastive_ssl/BYOL_SlowR50_8x8 | +| SimCLR | 2 | Slow-only | R50 | 8 x 8 | Kinetics-400 | 61.5 | 88.3 | 17.5 | 51.4 | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/videomoco_models/SimCLR_SlowR50_8x8_T2_epoch_00200.pyth) | contrastive_ssl/SimCLR_SlowR50_8x8 | +| SwAV | 2 | Slow-only | R50 | 8 x 8 | Kinetics-400 | 62.6 | 90.2 | 19.2 | 52.5| [`link`](https://dl.fbaipublicfiles.com/pyslowfast/videomoco_models/SwAV_SlowR50_8x8_T2_epoch_00200.pyth) | contrastive_ssl/SwAV_SlowR50_8x8 | +| MoCo | 4 | Slow-only | R50 | 8 x 8 | Kinetics-400 | 71.0 | 94.5 | 21.9 | 54.0 | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/videomoco_models/MoCo_SlowR50_8x8_T4_epoch_00200.pyth) | contrastive_ssl/MoCo_SlowR50_8x8 | +| BYOL | 4 | Slow-only | R50 | 8 x 8 | Kinetics-400 | 70.1 | 94.7 | xx | xx | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/videomoco_models/BYOL_SlowR50_8x8_T4_epoch_00200.pyth) | contrastive_ssl/BYOL_SlowR50_8x8 | +| BYOL | 4 | Slow-only | R50 | 16 x 4 | Kinetics-400 | 71.1 | 95.4 | xx | xx | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/videomoco_models/BYOL_SlowR50_16x4_T4_epoch_00200.pyth) | contrastive_ssl/BYOL_SlowR50_8x8 | ## Getting started -To use self-supervised learning techniques please refer to the configs under `configs/ssl`, or see the [MODEL_ZOO.md](https://github.com/facebookresearch/SlowFast/blob/master/MODEL_ZOO.md) for pre-trained models. See [paper](https://arxiv.org/abs/2104.14558) for details. For example, the command +To use self-supervised learning techniques please refer to the configs under `configs/contrastive_ssl`, or see the [MODEL_ZOO.md](https://github.com/facebookresearch/SlowFast/blob/master/MODEL_ZOO.md) for pre-trained models. See [paper](https://arxiv.org/abs/2104.14558) for details. For example, the command ``` python tools/run_net.py \ - --cfg configs/Kinetics/ssl/MoCo_SlowR50_8x8.yaml \ + --cfg configs/Kinetics/contrastive_ssl/MoCo_SlowR50_8x8.yaml \ DATA.PATH_TO_DATA_DIR path_to_your_dataset \ ``` @@ -40,6 +40,6 @@ If you find this useful for your research, please consider citing the paper usin @inproceedings{videossl2021, Author = {Christoph Feichtenhofer, Haoqi Fan, Bo Xiong, Ross Girshick, Kaiming He}, Title = {A Large-Scale Study on Unsupervised Spatiotemporal Representation Learning}, - Booktitle = {{CVPR}}, + Booktitle = {CVPR}, Year = {2021}} ``` diff --git a/projects/mae/README.md b/projects/mae/README.md new file mode 100644 index 000000000..c2a3d639e --- /dev/null +++ b/projects/mae/README.md @@ -0,0 +1,53 @@ +# Masked Autoencoders As Spatiotemporal Learners +[Christoph Feichtenhofer*](http://feichtenhofer.github.io/), [Haoqi Fan*](https://haoqifan.github.io/), [Yanghao Li](https://lyttonhao.github.io/), [Kaiming He](http://kaiminghe.com/) +
+Technical report, arXiv, May 2022. [[Paper](https://arxiv.org/abs/2205.09113)] +
+
+ +
+
+ +## Results & Models + +### **Kinetics-400**; configs are under configs/masked_ssl/ + + +| name | frame length x sample rate | top1 | Flops (G) x views | #params (M) | config pre-train (PT) | config fine-tune | model PT | +| ------------- | ------------- | ------------- | ------------- | ------------- | ------------- | ------------- | ------------- | +| ViT-B | 16 x 4 | 81.3 | 180 x 3 x 7 | 87 | k400_VIT_B_16x4_MAE_PT | k400_VIT_B_16x4_FT | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/masked_models/VIT_B_16x4_MAE_PT.pyth) | +| ViT-L | 16 x 4 | 84.8 | 598 x 3 x 7 | 304 | k400_VIT_L_16x4_MAE_PT | k400_VIT_L_16x4_FT | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/masked_models/VIT_L_16x4_MAE_PT.pyth) | +| ViT-H | 16 x 4 | 85.1 | 1193 x 3 x 7 | 632 | k400_VIT_H_16x4_MAE_PT | k400_VIT_H_16x4_FT | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/masked_models/VIT_H_16x4_MAE_PT_400e_85.1.pyth) | + + +## Getting started +To use self-supervised learning techniques please refer to the configs under `configs/masked_ssl`. For example, the command + +``` +python tools/run_net.py \ + --cfg configs/masked_ssl/k400_VIT_L_16x4_MAE_PT.yaml \ + DATA.PATH_TO_DATA_DIR path_to_your_Kinetics_dataset +``` + +should train an MAE ViT-L model on the Kinetics-400 dataset, and the command + +``` +python tools/run_net.py \ + --cfg configs/masked_ssl/k400_VIT_L_16x4_FT.yaml \ + DATA.PATH_TO_DATA_DIR path_to_your_Kinetics_dataset \ + TRAIN.CHECKPOINT_FILE_PATH path_to_your_pretrain_checkpoint +``` + +will fine-tune the resulting model, after passing the checkpoint path to the config. + + +## Reference +If you find this useful for your research, please consider citing the paper using the following BibTeX entry. +```BibTeX +@article{feichtenhofer2022masked, + title={Masked Autoencoders As Spatiotemporal Learners}, + author={Feichtenhofer, Christoph and Fan, Haoqi and Li, Yanghao and He, Kaiming}, + journal={arXiv preprint arXiv:2205.09113}, + year={2022} +} +``` diff --git a/projects/maskfeat/README.md b/projects/maskfeat/README.md new file mode 100644 index 000000000..0c1e4109a --- /dev/null +++ b/projects/maskfeat/README.md @@ -0,0 +1,81 @@ +# Masked Feature Prediction for Self-Supervised Visual Pre-Training +[Chen Wei*](https://weichen582.github.io/), [Haoqi Fan](https://haoqifan.github.io/), [Saining Xie](https://www.sainingxie.com/), [Chao-Yuan Wu](https://chaoyuan.org/), [Alan Yuille](https://www.cs.jhu.edu/~ayuille/), [Christoph Feichtenhofer*](http://feichtenhofer.github.io/) +
+In CVPR, 2022. [[Paper](https://arxiv.org/abs/2112.09133)] +
+
+ +
+
+ +## Results & Models + +### **ImageNet-1K**; configs are under configs/masked_ssl/ + + +| name | top1 | config pre-train (PT) | config fine-tune | model PT | +| ------------- | ------------- | ------------- | ------------- | ------------- | +| ViT-B | 81.3 | in1k_VIT_B_MaskFeat_PT | in1k_VIT_B_MaskFeat_FT | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/masked_models/in1k_VIT_B_MaskFeat_PT_epoch_01600.pyth) | +| ViT-L | 85.7 | in1k_VIT_L_MaskFeat_PT | in1k_VIT_L_MaskFeat_FT | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/masked_models/in1k_VIT_L_MaskFeat_PT_epoch_01600.pyth) | + + + +### **Kinetics-400**; configs are under configs/masked_ssl/ + + +| name | frame length x sample rate | top1 | Flops (G) x views | #params (M) | config pre-train (PT) | config fine-tune | model PT | +| ------------- | ------------- | ------------- | ------------- | ------------- | ------------- | ------------- | ------------- | +| MViT-S | 16 x 4 | 82.2 | 71 x 1 x 10 | 36 | k400_MVITv2_S_16x4_MaskFeat_PT | k400_MVITv2_S_16x4_FT | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/masked_models/k400_MVIT_S_MaskFeat_PT_epoch_00300.pyth) | +| MViT-L | 16 x 4 | 84.3 | 377 x 1 x 10 | 218 | k400_MVITv2_L_16x4_MaskFeat_PT | k400_MVITv2_L_16x4_FT | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/masked_models/k400_MVIT_L_MaskFeat_PT_epoch_00800.pyth) | + + + +## Getting started +To use self-supervised learning techniques please refer to the configs under `configs/masked_ssl`. For example, the command + +``` +python tools/run_net.py \ + --cfg configs/masked_ssl/k400_MVITv2_L_16x4_MaskFeat_PT.yaml \ + DATA.PATH_TO_DATA_DIR path_to_your_Kinetics_dataset +``` + +should train a MaskFeat MViT-L model on the Kinetics-400 dataset, and the command + +``` +python tools/run_net.py \ + --cfg configs/masked_ssl/k400_MVITv2_L_16x4_FT.yaml \ + DATA.PATH_TO_DATA_DIR path_to_your_Kinetics_dataset \ + TRAIN.CHECKPOINT_FILE_PATH path_to_your_pretrain_checkpoint +``` + +will fine-tune the resulting model, after passing the checkpoint path to the config. + +For images, the command + +``` +python tools/run_net.py \ + --cfg configs/masked_ssl/in1k_VIT_B_MaskFeat_PT.yaml \ + DATA.PATH_TO_DATA_DIR path_to_your_ImageNet_dataset +``` + +should train a MaskFeat ViT-B model on the ImageNet dataset, and the command + +``` +python tools/run_net.py \ + --cfg configs/masked_ssl/in1k_VIT_B_FT.yaml \ + DATA.PATH_TO_DATA_DIR path_to_your_ImageNet_dataset \ + TRAIN.CHECKPOINT_FILE_PATH path_to_your_pretrain_checkpoint +``` + +will fine-tune the resulting model, after passing the checkpoint path to the config. + +## Reference +If you find this useful for your research, please consider citing the paper using the following BibTeX entry. +```BibTeX +@InProceedings{wei2022masked, + author = {Wei, Chen and Fan, Haoqi and Xie, Saining and Wu, Chao-Yuan and Yuille, Alan and Feichtenhofer, Christoph}, + title = {Masked Feature Prediction for Self-Supervised Visual Pre-Training}, + booktitle = {CVPR}, + year = {2022}, +} +``` diff --git a/slowfast/config/defaults.py b/slowfast/config/defaults.py index 3be29ea03..f2fda219a 100644 --- a/slowfast/config/defaults.py +++ b/slowfast/config/defaults.py @@ -131,6 +131,10 @@ # If True Train the model, else skip training. _C.TRAIN.ENABLE = True +# Kill training if loss explodes over this ratio from the previous 5 measurements. +# Only enforced if > 0.0 +_C.TRAIN.KILL_LOSS_EXPLOSION_FACTOR = 0.0 + # Dataset. _C.TRAIN.DATASET = "kinetics" @@ -164,6 +168,9 @@ # If True, use FP16 for activations _C.TRAIN.MIXED_PRECISION = False +# if True, inflate some params from imagenet model. +_C.TRAIN.CHECKPOINT_IN_INIT = False + # ---------------------------------------------------------------------------- # # Augmentation options. # ---------------------------------------------------------------------------- # @@ -198,6 +205,32 @@ # Do not random erase first (clean) augmentation split. _C.AUG.RE_SPLIT = False +# Whether to generate input mask during image processing. +_C.AUG.GEN_MASK_LOADER = False + +# If True, masking mode is "tube". Default is "cube". +_C.AUG.MASK_TUBE = False + +# If True, masking mode is "frame". Default is "cube". +_C.AUG.MASK_FRAMES = False + +# The size of generated masks. +_C.AUG.MASK_WINDOW_SIZE = [8, 7, 7] + +# The ratio of masked tokens out of all tokens. Also applies to MViT supervised training +_C.AUG.MASK_RATIO = 0.0 + +# The maximum number of a masked block. None means no maximum limit. (Used only in image MaskFeat.) +_C.AUG.MAX_MASK_PATCHES_PER_BLOCK = None + +# ---------------------------------------------------------------------------- # +# Masked pretraining visualization options. +# ---------------------------------------------------------------------------- # +_C.VIS_MASK = CfgNode() + +# Whether to do visualization. +_C.VIS_MASK.ENABLE = False + # ---------------------------------------------------------------------------- # # MipUp options. # ---------------------------------------------------------------------------- # @@ -250,6 +283,8 @@ _C.TEST.CHECKPOINT_TYPE = "pytorch" # Path to saving prediction results file. _C.TEST.SAVE_RESULTS_PATH = "" + +_C.TEST.NUM_TEMPORAL_CLIPS = [] # ----------------------------------------------------------------------------- # ResNet options # ----------------------------------------------------------------------------- @@ -372,6 +407,7 @@ "slow", "x3d", "mvit", + "maskmvit", ] # Model architectures that has multiple pathways. @@ -444,6 +480,9 @@ # Drop path rate for the tranfomer. _C.MVIT.DROPPATH_RATE = 0.1 +# The initial value of layer scale gamma. Set 0.0 to disable layer scale. +_C.MVIT.LAYER_SCALE_INIT_VALUE = 0.0 + # Depth of the transformer. _C.MVIT.DEPTH = 16 @@ -460,7 +499,7 @@ # Stride size for the Pool KV at layer i. # Format: [[i, stride_t_i, stride_h_i, stride_w_i], ...,] -_C.MVIT.POOL_KV_STRIDE = None +_C.MVIT.POOL_KV_STRIDE = [] # Initial stride size for KV at layer 1. The stride size will be further reduced with # the raio of MVIT.DIM_MUL. If will overwrite MVIT.POOL_KV_STRIDE if not None. @@ -507,6 +546,66 @@ # If True, using separate linear layers for Q, K, V in attention blocks. _C.MVIT.SEPARATE_QKV = False +# The initialization scale factor for the head parameters. +_C.MVIT.HEAD_INIT_SCALE = 1.0 + +# Whether to use the mean pooling of all patch tokens as the output. +_C.MVIT.USE_MEAN_POOLING = False + +# If True, use frozen sin cos positional embedding. +_C.MVIT.USE_FIXED_SINCOS_POS = False + +# ----------------------------------------------------------------------------- +# Masked pretraining options +# ----------------------------------------------------------------------------- +_C.MASK = CfgNode() + +# Whether to enable Masked style pretraining. +_C.MASK.ENABLE = False + +# Whether to enable MAE (discard encoder tokens). +_C.MASK.MAE_ON = False + +# Whether to enable random masking in mae +_C.MASK.MAE_RND_MASK = False + +# Whether to do random masking per-frame in mae +_C.MASK.PER_FRAME_MASKING = False + +# only predict loss on temporal strided patches, or predict full time extent +_C.MASK.TIME_STRIDE_LOSS = True + +# Whether to normalize the pred pixel loss +_C.MASK.NORM_PRED_PIXEL = True + +# Whether to fix initialization with inverse depth of layer for pretraining. +_C.MASK.SCALE_INIT_BY_DEPTH = False + +# Base embedding dimension for the decoder transformer. +_C.MASK.DECODER_EMBED_DIM = 512 + +# Base embedding dimension for the decoder transformer. +_C.MASK.DECODER_SEP_POS_EMBED = False + +# Use a KV kernel in decoder? +_C.MASK.DEC_KV_KERNEL = [] + +# Use a KV stride in decoder? +_C.MASK.DEC_KV_STRIDE = [] + +# The depths of features which are inputs of the prediction head. +_C.MASK.PRETRAIN_DEPTH = [15] + +# The type of Masked pretraining prediction head. +# Can be "separate", "separate_xformer". +_C.MASK.HEAD_TYPE = "separate" + +# The depth of MAE's decoder +_C.MASK.DECODER_DEPTH = 0 + +# The weight of HOG target loss. +_C.MASK.PRED_HOG = False + # ----------------------------------------------------------------------------- # SlowFast options # ----------------------------------------------------------------------------- @@ -605,6 +704,9 @@ # Decoding backend, options include `pyav` or `torchvision` _C.DATA.DECODING_BACKEND = "torchvision" +# Decoding resize to short size (set to native size for best speed) +_C.DATA.DECODING_SHORT_SIZE = 256 + # if True, sample uniformly in [1 / max_scale, 1 / min_scale] and take a # reciprocal to get the scale. If False, take a uniform sample from # [min_scale, max_scale]. @@ -668,6 +770,15 @@ _C.DATA.SSL_BLUR_SIGMA_MAX = [0.0, 2.0] +# If combine train/val split as training for in21k +_C.DATA.IN22K_TRAINVAL = False + +# If not None, use IN1k as val split when training in21k +_C.DATA.IN22k_VAL_IN1K = "" + +# don't use real video for kinetics.py +_C.DATA.DUMMY_LOAD = False + # ---------------------------------------------------------------------------- # # Optimizer options # ---------------------------------------------------------------------------- # @@ -739,6 +850,11 @@ # LARS optimizer _C.SOLVER.LARS_ON = False +# The layer-wise decay of learning rate. Set to 1. to disable. +_C.SOLVER.LAYER_DECAY = 1.0 + +# Adam's beta +_C.SOLVER.BETAS = (0.9, 0.999) # ---------------------------------------------------------------------------- # # Misc options # ---------------------------------------------------------------------------- # diff --git a/slowfast/datasets/decoder.py b/slowfast/datasets/decoder.py index 7ebf713e3..7b7aff4ae 100644 --- a/slowfast/datasets/decoder.py +++ b/slowfast/datasets/decoder.py @@ -84,7 +84,7 @@ def get_multiple_start_end_idx( num_clips_uniform, min_delta=0, max_delta=math.inf, - use_offset=False + use_offset=False, ): """ Sample a clip of size clip_size from a video of size video_size and @@ -114,7 +114,7 @@ def sample_clips( min_delta=0, max_delta=math.inf, num_retries=100, - use_offset=False + use_offset=False, ): se_inds = np.empty((0, 2)) dt = np.empty((0)) @@ -125,13 +125,15 @@ def sample_clips( if clip_idx == -1: # Random temporal sampling. start_idx = random.uniform(0, max_start) - else: # Uniformly sample the clip with the given index. + else: # Uniformly sample the clip with the given index. if use_offset: if num_clips_uniform == 1: # Take the center clip if num_clips is 1. start_idx = math.floor(max_start / 2) else: - start_idx = clip_idx * math.floor(max_start / (num_clips_uniform - 1)) + start_idx = clip_idx * math.floor( + max_start / (num_clips_uniform - 1) + ) else: start_idx = max_start * clip_idx / num_clips_uniform @@ -302,10 +304,7 @@ def torchvision_decode( decode_all_video = False # try selective decoding clip_sizes = [ - np.maximum( - 1.0, - sampling_rate[i] * num_frames[i] / target_fps * fps - ) + np.maximum(1.0, sampling_rate[i] * num_frames[i] / target_fps * fps) for i in range(len(sampling_rate)) ] start_end_delta_time = get_multiple_start_end_idx( @@ -558,12 +557,9 @@ def decode( frames_decoded = [frames_decoded] num_decoded = len(frames_decoded) clip_sizes = [ - np.maximum( - 1.0, - sampling_rate[i] * num_frames[i] / target_fps * fps - ) - for i in range(len(sampling_rate)) - ] + np.maximum(1.0, sampling_rate[i] * num_frames[i] / target_fps * fps) + for i in range(len(sampling_rate)) + ] if decode_all_video: # full video was decoded (not trimmed yet) assert num_decoded == 1 and start_end_delta_time is None diff --git a/slowfast/datasets/imagenet.py b/slowfast/datasets/imagenet.py index 5ad6428f5..a2d45ba66 100644 --- a/slowfast/datasets/imagenet.py +++ b/slowfast/datasets/imagenet.py @@ -12,12 +12,13 @@ import slowfast.datasets.transform as transform import slowfast.utils.logging as logging +from slowfast.models.utils import calc_mvit_feature_geometry # import cv2 from slowfast.utils.env import pathmgr from .build import DATASET_REGISTRY -from .transform import transforms_imagenet_train +from .transform import MaskingGenerator, transforms_imagenet_train logger = logging.get_logger(__name__) @@ -42,6 +43,8 @@ def __init__(self, cfg, mode, num_retries=10): else: self._load_imdb() self.num_videos = len(self._imdb) + self.feat_size, self.feat_stride = calc_mvit_feature_geometry(cfg) + self.dummy_output = None def _load_imdb(self): split_path = os.path.join( @@ -117,8 +120,6 @@ def _prepare_im_res(self, im_path): im = transform.color_normalization( im, self.cfg.DATA.MEAN, self.cfg.DATA.STD ) - # Convert HWC/RGB/float to CHW/BGR/float format - # im = np.ascontiguousarray(im[:, :, ::-1].transpose([2, 0, 1])) return im def _prepare_im_tf(self, im_path): @@ -148,7 +149,7 @@ def _prepare_im_tf(self, im_path): size = int((256 / 224) * test_size) t.append( transforms_tv.Resize( - size, interpolation=3 + size, interpolation=transforms_tv.InterpolationMode.BICUBIC ), # to maintain same ratio w.r.t. 224 images ) t.append(transforms_tv.CenterCrop(test_size)) @@ -160,13 +161,59 @@ def _prepare_im_tf(self, im_path): im = aug_transform(im) return im + def _prepare_im_masked(self, im_path): + with pathmgr.open(im_path, "rb") as f: + with Image.open(f) as im: + im = im.convert("RGB") + + if self.mode in ["train", "val"]: + + depth = self.cfg.MASK.PRETRAIN_DEPTH[-1] + assert depth == max(self.cfg.MASK.PRETRAIN_DEPTH) + max_mask = self.cfg.AUG.MAX_MASK_PATCHES_PER_BLOCK + # use feat geometry for determining num masks + mask_window_size = self.feat_size[depth][-1] + num_mask = round( + self.feat_size[depth][-1] + * self.feat_size[depth][-2] + * self.cfg.AUG.MASK_RATIO + ) + min_mask = num_mask // 5 + + train_size = self.cfg.DATA.TRAIN_CROP_SIZE + mask_generator = MaskingGenerator( + mask_window_size, + num_masking_patches=num_mask, + max_num_patches=max_mask, + min_num_patches=min_mask, + ) + aug_transform = transforms_imagenet_train( + img_size=(train_size, train_size), + scale=self.cfg.DATA.TRAIN_JITTER_SCALES_RELATIVE, + ratio=self.cfg.DATA.TRAIN_JITTER_ASPECT_RELATIVE, + interpolation=self.cfg.AUG.INTERPOLATION, + color_jitter=self.cfg.AUG.COLOR_JITTER, + auto_augment=self.cfg.AUG.AA_TYPE, + re_prob=0.0, + mean=self.cfg.DATA.MEAN, + std=self.cfg.DATA.STD, + ) + im = aug_transform(im) + mask = mask_generator() + return [im, torch.Tensor(), mask] + else: + raise NotImplementedError + return aug_transform(im) + def __load__(self, index): try: # Load the image im_path = self._imdb[index]["im_path"] # Prepare the image for training / testing if self.cfg.AUG.ENABLE: - if self.mode == "train" and self.cfg.AUG.NUM_SAMPLE > 1: + if self.cfg.AUG.GEN_MASK_LOADER: + return self._prepare_im_masked(im_path) + elif self.mode == "train" and self.cfg.AUG.NUM_SAMPLE > 1: im = [] for _ in range(self.cfg.AUG.NUM_SAMPLE): crop = self._prepare_im_tf(im_path) @@ -182,24 +229,37 @@ def __load__(self, index): return None def __getitem__(self, index): + if self.dummy_output is not None: + return self.dummy_output # if the current image is corrupted, load a different image. for _ in range(self.num_retries): im = self.__load__(index) # Data corrupted, retry with a different image. if im is None: + assert self.mode == "train", f"{index} failed loading" + print(f"{index} failed. retry") index = random.randint(0, len(self._imdb) - 1) else: break # Retrieve the label label = self._imdb[index]["class"] if isinstance(im, list): - label = [label for _ in range(len(im))] - dummy = [torch.Tensor() for _ in range(len(im))] + if self.cfg.AUG.GEN_MASK_LOADER: + dummy = torch.Tensor() + label = torch.Tensor() + else: + label = [label for _ in range(len(im))] + dummy = [torch.Tensor() for _ in range(len(im))] + if self.cfg.DATA.DUMMY_LOAD: + if self.dummy_output is None: + self.dummy_output = (im, label, index, dummy, {}) return im, label, index, dummy, {} else: dummy = torch.Tensor() + if self.cfg.DATA.DUMMY_LOAD: + if self.dummy_output is None: + self.dummy_output = ([im], label, index, dummy, {}) return [im], label, index, dummy, {} - def __len__(self): return len(self._imdb) diff --git a/slowfast/datasets/kinetics.py b/slowfast/datasets/kinetics.py index 0e5eada55..c009bf812 100644 --- a/slowfast/datasets/kinetics.py +++ b/slowfast/datasets/kinetics.py @@ -18,7 +18,11 @@ from . import video_container as container from .build import DATASET_REGISTRY from .random_erasing import RandomErasing -from .transform import create_random_augment +from .transform import ( + MaskingGenerator, + MaskingGenerator3D, + create_random_augment, +) logger = logging.get_logger(__name__) @@ -74,6 +78,7 @@ def __init__(self, cfg, mode, num_retries=100): if self.mode in ["train"] and self.cfg.DATA.LOADER_CHUNK_SIZE > 0 else False ) + self.dummy_output = None # For training or validation mode, one single clip is sampled from every # video. For testing, NUM_ENSEMBLE_VIEWS clips are sampled from every # video. For every clip, NUM_SPATIAL_CROPS is cropped spatially from @@ -87,13 +92,14 @@ def __init__(self, cfg, mode, num_retries=100): logger.info("Constructing Kinetics {}...".format(mode)) self._construct_loader() - self.randaug = False + self.aug = False self.rand_erase = False self.use_temporal_gradient = False self.temporal_gradient_rate = 0.0 + self.cur_epoch = 0 if self.mode == "train" and self.cfg.AUG.ENABLE: - self.randaug = True + self.aug = True if self.cfg.AUG.RE_PROB > 0: self.rand_erase = True @@ -193,7 +199,8 @@ def __getitem__(self, index): index, self._num_yielded = index if self.cfg.MULTIGRID.SHORT_CYCLE: index, short_cycle_idx = index - + if self.dummy_output is not None: + return self.dummy_output if self.mode in ["train", "val"]: # -1 indicates random sampling. temporal_sample_index = -1 @@ -284,7 +291,10 @@ def __getitem__(self, index): self._path_to_videos[index], e ) ) - # Select a random video if the current video was not able to access. + if self.mode not in ["test"]: + # let's try another one + index = random.randint(0, len(self._path_to_videos) - 1) + continue # Select a random video if the current video was not able to access. if video_container is None: logger.warning( "Failed to meta load video idx {} from {}; trial {}".format( @@ -415,7 +425,7 @@ def __getitem__(self, index): gaussan_sigma_max=self.cfg.DATA.SSL_BLUR_SIGMA_MAX, ) - if self.randaug: + if self.aug and self.cfg.AUG.AA_TYPE: aug_transform = create_random_augment( input_size=(f_out[idx].size(1), f_out[idx].size(2)), auto_augment=self.cfg.AUG.AA_TYPE, @@ -478,19 +488,66 @@ def __getitem__(self, index): ).permute(1, 0, 2, 3) f_out[idx] = utils.pack_pathway_output(self.cfg, f_out[idx]) + if self.cfg.AUG.GEN_MASK_LOADER: + mask = self._gen_mask() + f_out[idx] = f_out[idx] + [torch.Tensor(), mask] frames = f_out[0] if num_out == 1 else f_out time_idx = np.array(time_idx_out) - if num_aug > 1: - label = [label] * num_aug - index = [index] * num_aug + if ( + num_aug * num_decode > 1 + and not self.cfg.MODEL.MODEL_NAME == "ContrastiveModel" + ): + label = [label] * num_aug * num_decode + index = [index] * num_aug * num_decode + if self.cfg.DATA.DUMMY_LOAD: + if self.dummy_output is None: + self.dummy_output = (frames, label, index, time_idx, {}) return frames, label, index, time_idx, {} else: - raise RuntimeError( - "Failed to fetch video idx {} from {}; after {} trials".format( - index, self._path_to_videos[index], i_try + logger.warning( + "Failed to fetch video after {} retries.".format( + self._num_retries ) ) + def _gen_mask(self): + if self.cfg.AUG.MASK_TUBE: + num_masking_patches = round( + np.prod(self.cfg.AUG.MASK_WINDOW_SIZE) * self.cfg.AUG.MASK_RATIO + ) + min_mask = num_masking_patches // 5 + masked_position_generator = MaskingGenerator( + mask_window_size=self.cfg.AUG.MASK_WINDOW_SIZE, + num_masking_patches=num_masking_patches, + max_num_patches=None, + min_num_patches=min_mask, + ) + mask = masked_position_generator() + mask = np.tile(mask, (8, 1, 1)) + elif self.cfg.AUG.MASK_FRAMES: + mask = np.zeros(shape=self.cfg.AUG.MASK_WINDOW_SIZE, dtype=np.int) + n_mask = round( + self.cfg.AUG.MASK_WINDOW_SIZE[0] * self.cfg.AUG.MASK_RATIO + ) + mask_t_ind = random.sample( + range(0, self.cfg.AUG.MASK_WINDOW_SIZE[0]), n_mask + ) + mask[mask_t_ind, :, :] += 1 + else: + num_masking_patches = round( + np.prod(self.cfg.AUG.MASK_WINDOW_SIZE) * self.cfg.AUG.MASK_RATIO + ) + max_mask = np.prod(self.cfg.AUG.MASK_WINDOW_SIZE[1:]) + min_mask = max_mask // 5 + masked_position_generator = MaskingGenerator3D( + mask_window_size=self.cfg.AUG.MASK_WINDOW_SIZE, + num_masking_patches=num_masking_patches, + max_num_patches=max_mask, + min_num_patches=min_mask, + ) + mask = masked_position_generator() + return mask + def _frame_to_list_img(self, frames): img_list = [ transforms.ToPILImage()(frames[i]) for i in range(frames.size(0)) diff --git a/slowfast/datasets/loader.py b/slowfast/datasets/loader.py index 71c4aaf66..2b06e8e18 100644 --- a/slowfast/datasets/loader.py +++ b/slowfast/datasets/loader.py @@ -148,13 +148,20 @@ def construct_loader(cfg, split, is_precise_bn=False): # Create a loader if cfg.DETECTION.ENABLE: collate_func = detection_collate - elif cfg.AUG.NUM_SAMPLE > 1 and split in ["train"]: + elif ( + ( + cfg.AUG.NUM_SAMPLE > 1 + or cfg.DATA.TRAIN_CROP_NUM_TEMPORAL > 1 + or cfg.DATA.TRAIN_CROP_NUM_SPATIAL > 1 + ) + and split in ["train"] + and not cfg.MODEL.MODEL_NAME == "ContrastiveModel" + ): collate_func = partial( multiple_samples_collate, fold="imagenet" in dataset_name ) else: collate_func = None - loader = torch.utils.data.DataLoader( dataset, batch_size=batch_size, diff --git a/slowfast/datasets/mixup.py b/slowfast/datasets/mixup.py index a75846d04..a52411883 100644 --- a/slowfast/datasets/mixup.py +++ b/slowfast/datasets/mixup.py @@ -181,8 +181,13 @@ def _mix_batch(self, x): return lam def __call__(self, x, target): - assert len(x) > 1, "Batch size should be greater than 1 for mixup." - lam = self._mix_batch(x) + if self.mix_prob > 0.0: + assert len(x) > 1, "Batch size should be greater than 1 for mixup." + lam = self._mix_batch(x) + elif self.mix_prob == 0.0: + lam = 1.0 + else: + raise NotImplementedError target = mixup_target( target, self.num_classes, lam, self.label_smoothing ) diff --git a/slowfast/datasets/transform.py b/slowfast/datasets/transform.py index a95b21a2e..e38f7c18d 100644 --- a/slowfast/datasets/transform.py +++ b/slowfast/datasets/transform.py @@ -799,6 +799,192 @@ def __repr__(self): return format_string +""" +This implementation is based on +https://github.com/microsoft/unilm/blob/master/beit/masking_generator.py +Licensed under The MIT License +""" + + +class MaskingGenerator: + def __init__( + self, + mask_window_size, + num_masking_patches, + min_num_patches=16, + max_num_patches=None, + min_aspect=0.3, + max_aspect=None, + ): + + if not isinstance( + mask_window_size, + ( + list, + tuple, + ), + ): + mask_window_size = (mask_window_size,) * 2 + self.height, self.width = mask_window_size + + self.num_patches = self.height * self.width + self.num_masking_patches = num_masking_patches + + self.min_num_patches = min_num_patches + self.max_num_patches = ( + num_masking_patches if max_num_patches is None else max_num_patches + ) + + max_aspect = max_aspect or 1 / min_aspect + self.log_aspect_ratio = (math.log(min_aspect), math.log(max_aspect)) + + def __repr__(self): + repr_str = "Generator(%d, %d -> [%d ~ %d], max = %d, %.3f ~ %.3f)" % ( + self.height, + self.width, + self.min_num_patches, + self.max_num_patches, + self.num_masking_patches, + self.log_aspect_ratio[0], + self.log_aspect_ratio[1], + ) + return repr_str + + def get_shape(self): + return self.height, self.width + + def _mask(self, mask, max_mask_patches): + delta = 0 + for _ in range(10): + target_area = random.uniform(self.min_num_patches, max_mask_patches) + aspect_ratio = math.exp(random.uniform(*self.log_aspect_ratio)) + h = int(round(math.sqrt(target_area * aspect_ratio))) + w = int(round(math.sqrt(target_area / aspect_ratio))) + if w < self.width and h < self.height: + top = random.randint(0, self.height - h) + left = random.randint(0, self.width - w) + + num_masked = mask[top : top + h, left : left + w].sum() + # Overlap + if 0 < h * w - num_masked <= max_mask_patches: + for i in range(top, top + h): + for j in range(left, left + w): + if mask[i, j] == 0: + mask[i, j] = 1 + delta += 1 + + if delta > 0: + break + return delta + + def __call__(self): + mask = np.zeros(shape=self.get_shape(), dtype=np.int) + mask_count = 0 + while mask_count < self.num_masking_patches: + max_mask_patches = self.num_masking_patches - mask_count + max_mask_patches = min(max_mask_patches, self.max_num_patches) + + delta = self._mask(mask, max_mask_patches) + if delta == 0: + break + else: + mask_count += delta + + return mask + + +""" +This implementation is based on +https://github.com/microsoft/unilm/blob/master/beit/masking_generator.py +Licensed under The MIT License +""" + + +class MaskingGenerator3D: + def __init__( + self, + mask_window_size, + num_masking_patches, + min_num_patches=16, + max_num_patches=None, + min_aspect=0.3, + max_aspect=None, + ): + + self.temporal, self.height, self.width = mask_window_size + self.num_masking_patches = num_masking_patches + self.min_num_patches = min_num_patches + self.max_num_patches = ( + num_masking_patches if max_num_patches is None else max_num_patches + ) + max_aspect = max_aspect or 1 / min_aspect + self.log_aspect_ratio = (math.log(min_aspect), math.log(max_aspect)) + + def __repr__(self): + repr_str = ( + "Generator(%d, %d, %d -> [%d ~ %d], max = %d, %.3f ~ %.3f)" + % ( + self.temporal, + self.height, + self.width, + self.min_num_patches, + self.max_num_patches, + self.num_masking_patches, + self.log_aspect_ratio[0], + self.log_aspect_ratio[1], + ) + ) + return repr_str + + def get_shape(self): + return self.temporal, self.height, self.width + + def _mask(self, mask, max_mask_patches): + delta = 0 + for _ in range(100): + target_area = random.uniform( + self.min_num_patches, self.max_num_patches + ) + aspect_ratio = math.exp(random.uniform(*self.log_aspect_ratio)) + h = int(round(math.sqrt(target_area * aspect_ratio))) + w = int(round(math.sqrt(target_area / aspect_ratio))) + t = random.randint(1, self.temporal) # ! + if w < self.width and h < self.height: + top = random.randint(0, self.height - h) + left = random.randint(0, self.width - w) + front = random.randint(0, self.temporal - t) + + num_masked = mask[ + front : front + t, top : top + h, left : left + w + ].sum() + # Overlap + if 0 < h * w * t - num_masked <= max_mask_patches: + for i in range(front, front + t): + for j in range(top, top + h): + for k in range(left, left + w): + if mask[i, j, k] == 0: + mask[i, j, k] = 1 + delta += 1 + + if delta > 0: + break + return delta + + def __call__(self): + mask = np.zeros(shape=self.get_shape(), dtype=np.int) + mask_count = 0 + while mask_count < self.num_masking_patches: + max_mask_patches = self.num_masking_patches - mask_count + + delta = self._mask(mask, max_mask_patches) + if delta == 0: + break + else: + mask_count += delta + + return mask + + def transforms_imagenet_train( img_size=224, scale=None, diff --git a/slowfast/models/__init__.py b/slowfast/models/__init__.py index 88e806198..12a6a8f44 100644 --- a/slowfast/models/__init__.py +++ b/slowfast/models/__init__.py @@ -4,7 +4,8 @@ from .build import MODEL_REGISTRY, build_model # noqa from .contrastive import ContrastiveModel # noqa from .custom_video_model_builder import * # noqa -from .video_model_builder import ResNet, SlowFast # noqa +from .masked import MaskMViT # noqa +from .video_model_builder import MViT, ResNet, SlowFast # noqa try: from .ptv_model_builder import ( diff --git a/slowfast/models/attention.py b/slowfast/models/attention.py index 22abc77a9..a8c8ec8a0 100644 --- a/slowfast/models/attention.py +++ b/slowfast/models/attention.py @@ -193,6 +193,7 @@ def __init__( head_dim = dim_out // num_heads self.scale = head_dim**-0.5 self.has_cls_embed = has_cls_embed + self.mode = mode padding_q = [int(q // 2) for q in kernel_q] padding_kv = [int(kv // 2) for kv in kernel_kv] @@ -212,7 +213,6 @@ def __init__( kernel_q = () if numpy.prod(kernel_kv) == 1 and numpy.prod(stride_kv) == 1: kernel_kv = () - self.mode = mode if mode in ("avg", "max"): pool_op = nn.MaxPool3d if mode == "max" else nn.AvgPool3d @@ -454,6 +454,7 @@ def __init__( qk_scale=None, drop_rate=0.0, drop_path=0.0, + layer_scale_init_value=0.0, act_layer=nn.GELU, norm_layer=nn.LayerNorm, up_rate=None, @@ -519,6 +520,17 @@ def __init__( act_layer=act_layer, drop_rate=drop_rate, ) + if layer_scale_init_value > 0: + self.gamma_1 = nn.Parameter( + layer_scale_init_value * torch.ones((dim)), requires_grad=True + ) + self.gamma_2 = nn.Parameter( + layer_scale_init_value * torch.ones((dim_out)), + requires_grad=True, + ) + else: + self.gamma_1, self.gamma_2 = None, None + if dim != dim_out: self.proj = nn.Linear(dim, dim_out) @@ -526,11 +538,11 @@ def __init__( nn.MaxPool3d( kernel_skip, stride_skip, padding_skip, ceil_mode=False ) - if len(kernel_skip) > 0 + if len(stride_skip) > 0 and numpy.prod(stride_skip) > 1 else None ) - def forward(self, x, thw_shape): + def forward(self, x, thw_shape=None): x_norm = self.norm1(x) x_block, thw_shape_new = self.attn(x_norm, thw_shape) if self.dim_mul_in_att and self.dim != self.dim_out: @@ -538,10 +550,19 @@ def forward(self, x, thw_shape): x_res, _ = attention_pool( x, self.pool_skip, thw_shape, has_cls_embed=self.has_cls_embed ) - x = x_res + self.drop_path(x_block) + if self.gamma_1 is not None: + x = x_res + self.drop_path(self.gamma_1 * x_block) + else: + x = x_res + self.drop_path(x_block) x_norm = self.norm2(x) x_mlp = self.mlp(x_norm) if not self.dim_mul_in_att and self.dim != self.dim_out: x = self.proj(x_norm) - x = x + self.drop_path(x_mlp) - return x, thw_shape_new + if self.gamma_2 is not None: + x = x + self.drop_path(self.gamma_2 * x_mlp) + else: + x = x + self.drop_path(x_mlp) + if thw_shape: + return x, thw_shape_new + else: + return x diff --git a/slowfast/models/contrastive.py b/slowfast/models/contrastive.py index 516e783b6..57deb3336 100644 --- a/slowfast/models/contrastive.py +++ b/slowfast/models/contrastive.py @@ -1,8 +1,6 @@ #!/usr/bin/env python3 # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. -"""CSC model.""" - import math import numpy as np import torch @@ -1082,7 +1080,8 @@ def update(self, mem, momentum, ind, time, interp=False): return -def cancel_swav_gradients(model, cfg, epoch_exact): +def contrastive_parameter_surgery(model, cfg, epoch_exact, cur_iter): + # cancel some gradients in first epoch of SwAV if ( cfg.MODEL.MODEL_NAME == "ContrastiveModel" @@ -1092,4 +1091,76 @@ def cancel_swav_gradients(model, cfg, epoch_exact): for name, p in model.named_parameters(): if "swav_prototypes" in name: p.grad = None - return model + + iters_noupdate = 0 + if ( + cfg.MODEL.MODEL_NAME == "ContrastiveModel" + and cfg.CONTRASTIVE.TYPE == "moco" + ): + assert ( + cfg.CONTRASTIVE.QUEUE_LEN % (cfg.TRAIN.BATCH_SIZE * cfg.NUM_SHARDS) + == 0 + ) + iters_noupdate = ( + cfg.CONTRASTIVE.QUEUE_LEN // cfg.TRAIN.BATCH_SIZE // cfg.NUM_SHARDS + ) + + if cur_iter < iters_noupdate and epoch_exact < 1: # for e.g. MoCo + logger.info( + "Not updating parameters {}/{}".format(cur_iter, iters_noupdate) + ) + update_param = False + else: + update_param = True + + return model, update_param + + +def contrastive_forward(model, cfg, inputs, index, time, epoch_exact, scaler): + if cfg.CONTRASTIVE.SEQUENTIAL: + perform_backward = False + mdl = model.module if hasattr(model, "module") else model + keys = ( + mdl.compute_key_feat( + inputs, + compute_predictor_keys=False, + batched_inference=True if len(inputs) < 2 else False, + ) + if cfg.CONTRASTIVE.TYPE == "moco" or cfg.CONTRASTIVE.TYPE == "byol" + else [None] * len(inputs) + ) + for k, vid in enumerate(inputs): + other_keys = keys[:k] + keys[k + 1 :] + time_cur = torch.cat( + [ + time[:, k : k + 1, :], + time[:, :k, :], + time[:, k + 1 :, :], + ], + 1, + ) # q, kpre, kpost + vids = [vid] + if ( + cfg.CONTRASTIVE.TYPE == "swav" + or cfg.CONTRASTIVE.TYPE == "simclr" + ): + if k < len(inputs) - 1: + vids = inputs[k : k + 2] + else: + break + lgt_k, loss_k = model( + vids, index, time_cur, epoch_exact, keys=other_keys + ) + scaler.scale(loss_k).backward() + if k == 0: + preds, partial_loss = lgt_k, loss_k.detach() + else: + preds = torch.cat([preds, lgt_k], dim=0) + partial_loss += loss_k.detach() + partial_loss /= len(inputs) * 2.0 # to have same loss as symm model + if cfg.CONTRASTIVE.TYPE == "moco": + mdl._dequeue_and_enqueue(keys) + else: + perform_backward = True + preds, partial_loss = model(inputs, index, time, epoch_exact, keys=None) + return model, preds, partial_loss, perform_backward diff --git a/slowfast/models/head_helper.py b/slowfast/models/head_helper.py index 937208a7b..9d56f3374 100644 --- a/slowfast/models/head_helper.py +++ b/slowfast/models/head_helper.py @@ -3,15 +3,20 @@ """ResNe(X)t Head helper.""" +from functools import partial import torch import torch.nn as nn from detectron2.layers import ROIAlign +import slowfast.utils.logging as logging +from slowfast.models.attention import MultiScaleBlock from slowfast.models.batchnorm_helper import ( NaiveSyncBatchNorm1d as NaiveSyncBatchNorm1d, ) from slowfast.models.nonlocal_helper import Nonlocal +logger = logging.get_logger(__name__) + class ResNetRoIHead(nn.Module): """ @@ -570,3 +575,116 @@ def forward(self, x): x = x.view(x.shape[0], -1) return x + + +class MSSeparateHead(nn.Module): + """ + Perform linear projection or Transformer-based decoder (optionally MultiScale) + for mask prediction models. + Args: + blocks (MultiScaleBlock): the encoder blocks to provide input dimensions of the head. + num_classes (int): the dimension of the prediction target (eg. HOG or pixels). + feat_sz (list): the spatiotemporal sizes of the input features. + """ + + def __init__( + self, + blocks, + cfg, + num_classes, + feat_sz, + ): + super(MSSeparateHead, self).__init__() + head_type = cfg.MASK.HEAD_TYPE.split("_") + assert head_type[0] == "separate" + if len(head_type) > 1: + transform_type = head_type[1] + assert transform_type in ["xformer"] + else: + transform_type = None + + depth_list = cfg.MASK.PRETRAIN_DEPTH + mlp_ratio = cfg.MVIT.MLP_RATIO + qkv_bias = cfg.MVIT.QKV_BIAS + drop_rate = cfg.MVIT.DROPOUT_RATE + kernel_kv = cfg.MASK.DEC_KV_KERNEL + stride_kv = cfg.MASK.DEC_KV_STRIDE + mode = cfg.MVIT.MODE + self.cls_embed_on = cfg.MVIT.CLS_EMBED_ON + pool_first = cfg.MVIT.POOL_FIRST + if cfg.MVIT.NORM == "layernorm": + norm_layer = partial(nn.LayerNorm, eps=1e-6) + else: + raise NotImplementedError("Only supports layernorm.") + + self.transforms = nn.ModuleList() + self.projections = nn.ModuleList() + for depth, num_class, feature_size in zip( + depth_list, num_classes, feat_sz + ): + head_dim = ( + cfg.MASK.DECODER_EMBED_DIM + if cfg.MASK.MAE_ON + else blocks[depth].dim_out + ) + op = [] + if transform_type == "xformer": + assert cfg.MASK.DECODER_DEPTH > 0 + for _ in range(cfg.MASK.DECODER_DEPTH): + dim_out = cfg.MASK.DECODER_EMBED_DIM + op.append( + MultiScaleBlock( + dim=head_dim, + dim_out=dim_out, + input_size=feature_size, + num_heads=dim_out // 64, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + drop_rate=drop_rate, + drop_path=0.0, + norm_layer=norm_layer, + kernel_q=[], + kernel_kv=kernel_kv, + stride_q=[], + stride_kv=stride_kv, + mode=mode, + has_cls_embed=self.cls_embed_on, + pool_first=pool_first, + ) + ) + head_dim = dim_out + + op.append(norm_layer(head_dim)) + self.transforms.append(nn.Sequential(*op)) + self.projections.append(nn.Linear(head_dim, num_class, bias=True)) + + self.apply(self._init_weights) + + def _init_weights(self, m): + if isinstance(m, nn.Linear): + nn.init.trunc_normal_(m.weight, std=0.02) + # FYI: MAE uses xavier_uniform following official JAX ViT: + # torch.nn.init.xavier_uniform_(m.weight) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + + def forward(self, block_outputs, output_masks, return_all, thw): + model_outputs = [] + for idx, x in enumerate(block_outputs): + for _, blk in enumerate(self.transforms[idx]): + if isinstance(blk, MultiScaleBlock): + x, thw = blk(x, thw) + else: + x = blk(x) + + if self.cls_embed_on: + x = x[:, 1:] + if not return_all: + mask = output_masks[idx] + x = x[mask] + x = self.projections[idx](x) + model_outputs.append(x) + return model_outputs diff --git a/slowfast/models/losses.py b/slowfast/models/losses.py index 96e39b5f4..7dd79fb0f 100644 --- a/slowfast/models/losses.py +++ b/slowfast/models/losses.py @@ -4,7 +4,6 @@ """Loss functions.""" from functools import partial - import torch import torch.nn as nn @@ -26,12 +25,53 @@ def forward(self, inputs, dummy_labels=None): return loss +class MultipleMSELoss(nn.Module): + """ + Compute multiple mse losses and return their average. + """ + + def __init__(self, reduction="mean"): + """ + Args: + reduction (str): specifies reduction to apply to the output. It can be + "mean" (default) or "none". + """ + super(MultipleMSELoss, self).__init__() + self.mse_func = nn.MSELoss(reduction=reduction) + + def forward(self, x, y): + loss_sum = 0.0 + multi_loss = [] + for xt, yt in zip(x, y): + if isinstance(yt, (tuple,)): + if len(yt) == 2: + yt, wt = yt + lt = "mse" + elif len(yt) == 3: + yt, wt, lt = yt + else: + raise NotImplementedError + else: + wt, lt = 1.0, "mse" + if lt == "mse": + loss = self.mse_func(xt, yt) + else: + raise NotImplementedError + loss_sum += loss * wt + multi_loss.append(loss) + return loss_sum, multi_loss + + _LOSSES = { "cross_entropy": nn.CrossEntropyLoss, "bce": nn.BCELoss, "bce_logit": nn.BCEWithLogitsLoss, - "soft_cross_entropy": partial(SoftTargetCrossEntropyLoss, normalize_targets=False), + "soft_cross_entropy": partial( + SoftTargetCrossEntropyLoss, normalize_targets=False + ), "contrastive_loss": ContrastiveLoss, + "mse": nn.MSELoss, + "multi_mse": MultipleMSELoss, } diff --git a/slowfast/models/masked.py b/slowfast/models/masked.py new file mode 100644 index 000000000..d60c8a0bc --- /dev/null +++ b/slowfast/models/masked.py @@ -0,0 +1,658 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +import math +from functools import partial +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.nn.init import trunc_normal_ + +import slowfast.utils.logging as logging +import slowfast.utils.misc as misc +from slowfast.models import head_helper +from slowfast.models.attention import attention_pool +from slowfast.models.utils import calc_mvit_feature_geometry +from slowfast.models.video_model_builder import MViT + +from . import head_helper, operators, resnet_helper, stem_helper # noqa +from .build import MODEL_REGISTRY + +logger = logging.get_logger(__name__) + + +@MODEL_REGISTRY.register() +class MaskMViT(MViT): + def __init__(self, cfg): + super().__init__(cfg) + self.pretrain_depth = cfg.MASK.PRETRAIN_DEPTH + if self.pretrain_depth[-1] + 1 < cfg.MVIT.DEPTH: + del self.blocks[self.pretrain_depth[-1] + 1 :] + del self.norm + del self.head + self.feat_size, self.feat_stride = calc_mvit_feature_geometry(cfg) + + self.head_type = cfg.MASK.HEAD_TYPE.split("_") + feat_sz = [self.feat_size[depth] for depth in self.pretrain_depth] + if self.head_type[0] == "separate": + if not cfg.MASK.PRED_HOG: + pred_t_sz = ( + 1 + if self.cfg.MASK.TIME_STRIDE_LOSS + else self.patch_stride[0] + ) + num_classes = [ + pred_t_sz * (self.feat_stride[depth][-1] ** 2) * 3 + for depth in self.pretrain_depth + ] + self.pred_head = head_helper.MSSeparateHead( + self.blocks, cfg, num_classes, feat_sz + ) + else: + self.hogs = nn.ModuleList() + self.nbins = 9 + self.cell_sz = 8 + self.hogs.append( + operators.HOGLayerC( + nbins=self.nbins, + pool=self.cell_sz, + ) + ) + self.ncells = [ + (self.feat_stride[depth][-1] // self.cell_sz) ** 2 + for depth in self.pretrain_depth + ] + pred_hog_classes = [self.nbins * ncell for ncell in self.ncells] + pred_hog_classes = [ + pred_hog_class * 3 # 3 color channels + for pred_hog_class in pred_hog_classes + ] + self.pred_head = head_helper.MSSeparateHead( + self.blocks, cfg, pred_hog_classes, feat_sz + ) + self.hog_loss = "mse" + else: + raise NotImplementedError + + embed_dim = cfg.MVIT.EMBED_DIM + decoder_embed_dim = cfg.MASK.DECODER_EMBED_DIM + self.sep_pos_embed_decoder = cfg.MASK.DECODER_SEP_POS_EMBED + self.counter = 0 + if cfg.MASK.MAE_ON: + # ---------------------------------------------------------------- + # MAE decoder specifics + norm_layer = partial(nn.LayerNorm, eps=1e-6) + dim_in = self.blocks[-1].dim_out + self.norm = norm_layer(dim_in) + self.decoder_embed = nn.Linear(dim_in, decoder_embed_dim, bias=True) + num_patches = math.prod(self.patch_dims) + if self.use_abs_pos: + if self.sep_pos_embed_decoder: + self.dec_pos_embed_spatial = nn.Parameter( + torch.zeros( + 1, + self.patch_dims[1] * self.patch_dims[2], + decoder_embed_dim, + ) + ) + self.dec_pos_embed_temporal = nn.Parameter( + torch.zeros(1, self.patch_dims[0], decoder_embed_dim) + ) + if self.cls_embed_on: + self.dec_pos_embed_class = nn.Parameter( + torch.zeros(1, 1, decoder_embed_dim) + ) + else: + self.decoder_pos_embed = nn.Parameter( + torch.zeros( + 1, + num_patches + 1 + if self.cls_embed_on + else num_patches, + decoder_embed_dim, + ) + ) + self.mask_token = nn.Parameter( + torch.zeros( + 1, 1, decoder_embed_dim if cfg.MASK.MAE_ON else embed_dim + ) + ) + trunc_normal_(self.mask_token, std=0.02) + if self.use_abs_pos and cfg.MASK.MAE_ON: + if self.sep_pos_embed_decoder: + trunc_normal_(self.dec_pos_embed_spatial, std=0.02) + trunc_normal_(self.dec_pos_embed_temporal, std=0.02) + if self.cls_embed_on: + trunc_normal_(self.dec_pos_embed_class, std=0.02) + else: + trunc_normal_(self.decoder_pos_embed, std=0.02) + + if cfg.MASK.SCALE_INIT_BY_DEPTH: + self.fix_init_weight() + + self.pred_pixel_wt = 0.0 if cfg.MASK.PRED_HOG else 1.0 + self.pred_hog_wt = 1.0 if cfg.MASK.PRED_HOG else 0.0 + + @torch.jit.ignore + def no_weight_decay(self): + names = [] + if self.cfg.MVIT.ZERO_DECAY_POS_CLS: + if self.use_abs_pos: + if self.sep_pos_embed_decoder: + names.extend( + [ + "dec_pos_embed_spatial", + "dec_pos_embed_temporal", + "dec_pos_embed_class", + ] + ) + else: + names.extend(["pos_embed_decoder"]) + if self.cls_embed_on: + names.append("cls_token") + + return names + + def fix_init_weight(self): + def rescale(param, layer_id): + param.div_(math.sqrt(2.0 * layer_id)) + + for layer_id, layer in enumerate(self.blocks): + rescale(layer.attn.proj.weight.data, layer_id + 1) + rescale(layer.mlp.fc2.weight.data, layer_id + 1) + for trans in self.pred_head.transforms: + for layer_id, layer in enumerate(trans): + if hasattr(layer, "attn"): + rescale( + layer.attn.proj.weight.data, + layer_id + 1 + len(self.blocks), + ) # or + len(self.blocks) + rescale(layer.mlp.fc2.weight.data, layer_id + 1) + + def _get_multiscale_mask(self, mask): + if self.use_2d_patch: + mask = mask.unsqueeze(0) + output_masks = [] + for depth in self.pretrain_depth: + size = self.feat_size[depth][-1] + output_mask = F.interpolate(mask, size=size) + if self.use_2d_patch: + output_mask = output_mask[0] + output_mask = output_mask.flatten(1).to(torch.bool) + output_masks.append(output_mask) + return output_masks + + def _patchify(self, imgs, p=16, time_stride_loss=True): + N, _, T, H, W = imgs.shape + u = 1 if time_stride_loss else self.patch_stride[0] + assert H == W and H % p == 0 and T % u == 0 + h = w = H // p + t = T // u + x = imgs.reshape(shape=(N, 3, t, u, h, p, w, p)) + x = torch.einsum("nctuhpwq->nthwupqc", x) + x = x.reshape(shape=(N, t * h * w, u * p**2 * 3)) + self.patch_info = (N, T, H, W, p, u, t, h, w) + return x + + def _unpatchify(self, x): + N, T, H, W, p, u, t, h, w = self.patch_info + x = x.reshape(shape=(N, t, h, w, u, p, p, 3)) + x = torch.einsum("nthwupqc->nctuhpwq", x) + imgs = x.reshape(shape=(N, 3, T, H, W)) + return imgs + + def _get_pixel_label_2d(self, input_img, output_masks, norm=True): + input_img = input_img.permute(0, 2, 3, 1) + labels = [] + for depth, output_mask in zip(self.pretrain_depth, output_masks): + size = self.feat_stride[depth][-1] + label = input_img.unfold(1, size, size).unfold(2, size, size) + label = label.flatten(1, 2).flatten(2) + label = label[output_mask] + if norm: + mean = label.mean(dim=-1, keepdim=True) + var = label.var(dim=-1, keepdim=True) + label = (label - mean) / (var + 1.0e-6) ** 0.5 + labels.append(label) + return labels + + def _get_pixel_label_3d( + self, input_frames, output_masks, time_stride_loss=True, norm=True + ): + if time_stride_loss: + input_frames = input_frames[ + :, :, :: self.cfg.MVIT.PATCH_STRIDE[0], :, : + ] + imgs = input_frames + input_frames = input_frames.permute(0, 2, 3, 4, 1) + labels = [] + for depth, output_mask in zip(self.pretrain_depth, output_masks): + size = self.feat_stride[depth][-1] + label = self._patchify( + imgs, p=size, time_stride_loss=time_stride_loss + ) + label = label[output_mask] + + if norm: # self.norm_pix_loss: + mean = label.mean(dim=-1, keepdim=True) + var = label.var(dim=-1, keepdim=True) + label = (label - mean) / (var + 1.0e-6) ** 0.5 + labels.append( + (label, self.pred_pixel_wt / len(self.pretrain_depth)) + ) + return labels + + def _get_hog_label_2d(self, input_frames, output_masks): + # input_frames, B C H W + labels = [] + for depth, output_mask in zip(self.pretrain_depth, output_masks): + feat_size = self.feat_size[depth][-1] + hog_list = [] + for hog in self.hogs: + tmp_hog = hog(input_frames).flatten(1, 2) # return B C H W + unfold_size = tmp_hog.shape[-1] // feat_size + tmp_hog = ( + tmp_hog.permute(0, 2, 3, 1) + .unfold(1, unfold_size, unfold_size) + .unfold(2, unfold_size, unfold_size) + .flatten(1, 2) + .flatten(2) + ) + tmp_hog = tmp_hog[output_mask] + hog_list.append(tmp_hog) + all_tlabel = torch.cat(hog_list, -1) + labels.append((all_tlabel, self.pred_hog_wt, self.hog_loss)) + return labels + + def _get_hog_label_3d(self, input_frames, output_masks): + input_frames = input_frames[ + :, :, :: self.cfg.MVIT.PATCH_STRIDE[0], :, : + ] # B C T H W + input_frames = input_frames.transpose(1, 2) # B T C H W + B, T = input_frames.shape[:2] + input_frames = input_frames.flatten(0, 1) # BT C H W + labels = [] + for depth, output_mask in zip(self.pretrain_depth, output_masks): + feat_size = self.feat_size[depth][-1] + hog_list = [] + for hog in self.hogs: + tmp_hog = hog(input_frames).flatten(1, 2) # BT C H W + unfold_size = tmp_hog.shape[-1] // feat_size + tmp_hog = ( + tmp_hog.permute(0, 2, 3, 1) + .unfold(1, unfold_size, unfold_size) + .unfold(2, unfold_size, unfold_size) + ) # BT h w C wh ww + tmp_hog = tmp_hog.flatten(3).view( + B, T, feat_size, feat_size, -1 + ) # B T h w C (3 nbins h w) + tmp_hog = tmp_hog.flatten(1, 3) # B N C + tmp_hog = tmp_hog[output_mask] + hog_list.append(tmp_hog) + all_tlabel = torch.cat(hog_list, -1) + labels.append((all_tlabel, self.pred_hog_wt, self.hog_loss)) + return labels + + def _mae_random_masking(self, x, mask_ratio, mask_in=None): + """ + Perform per-sample random masking by per-sample shuffling. + Per-sample shuffling is done by argsort random noise. + x: [N, L, D], sequence + """ + N, L, D = x.shape # batch, length, dim + if mask_in is None: + if self.cfg.AUG.MASK_TUBE: + noise = ( + torch.rand(N, 1, self.H * self.W, device=x.device) + .repeat([1, self.T, 1]) + .reshape(N, L) + ) # noise in [0, 1] + else: + noise = torch.rand(N, L, device=x.device) # noise in [0, 1] + else: + noise = mask_in.flatten(1) + mask_ratio = sum(noise.flatten()) / noise.numel() # alrdy masked + len_keep = int(L * (1 - mask_ratio)) + assert len_keep > 1 + # sort noise for each sample + ids_shuffle = torch.argsort( + noise, dim=1 + ) # ascend: small is keep, large is remove + ids_restore = torch.argsort(ids_shuffle, dim=1) + # keep the first subset + ids_keep = ids_shuffle[:, :len_keep] + x_masked = torch.gather( + x, dim=1, index=ids_keep.unsqueeze(-1).repeat(1, 1, D) + ) + # generate the binary mask: 0 is keep, 1 is remove + mask = torch.ones([N, L], device=x.device) + mask[:, :len_keep] = 0 + # unshuffle to get the binary mask + mask = torch.gather(mask, dim=1, index=ids_restore) + return x_masked, mask, ids_restore, ids_keep + + def _mae_forward_encoder(self, x, mask_ratio, mask=None): + x, bcthw = self.patch_embed(x, keep_spatial=False) + bcthw = list(bcthw) + if len(bcthw) == 4: # Fix bcthw in case of 4D tensor + bcthw.insert(2, torch.tensor(self.T)) + T, H, W = bcthw[-3], bcthw[-2], bcthw[-1] + assert len(bcthw) == 5 and (T, H, W) == (self.T, self.H, self.W), bcthw + s = 1 if self.cls_embed_on else 0 + B, N, C = x.shape + + if self.use_fixed_sincos_pos: + x += self.pos_embed[:, s:, :] # 0: no cls token + + if self.cfg.MASK.PER_FRAME_MASKING: + x = x.reshape([B * T, H * W, C]) + x, mask, ids_restore, ids_keep = self._mae_random_masking( + x, mask_ratio, None if self.cfg.MASK.MAE_RND_MASK else mask + ) + if self.cfg.MASK.PER_FRAME_MASKING: + x = x.view([B, -1, C]) + + if self.cls_embed_on: + # append cls token + cls_token = self.cls_token # + if self.use_fixed_sincos_pos: + cls_token = cls_token + self.pos_embed[:, :s, :] + cls_tokens = cls_token.expand(x.shape[0], -1, -1) + x = torch.cat((cls_tokens, x), dim=1) + + if self.use_abs_pos and not self.use_fixed_sincos_pos: + if self.sep_pos_embed: + pos_embed = self.pos_embed_spatial.repeat( + 1, self.patch_dims[0], 1 + ) + torch.repeat_interleave( + self.pos_embed_temporal, + self.patch_dims[1] * self.patch_dims[2], + dim=1, + ) + pos_embed = pos_embed.expand(x.shape[0], -1, -1) + pos_embed = torch.gather( + pos_embed, + dim=1, + index=ids_keep.unsqueeze(-1).repeat( + 1, 1, pos_embed.shape[2] + ), + ) + if self.cls_embed_on: + pos_embed = torch.cat( + [ + self.pos_embed_class.expand( + pos_embed.shape[0], -1, -1 + ), + pos_embed, + ], + 1, + ) + x += pos_embed + else: + pos_embed = self.pos_embed.expand(x.shape[0], -1, -1) + pos_embed_sampled = torch.gather( + pos_embed[:, s:, :], + dim=1, + index=ids_keep.unsqueeze(-1).repeat( + 1, 1, self.pos_embed.shape[2] + ), + ) + if self.cls_embed_on: + pos_embed_sampled = torch.cat( + [pos_embed[:, :s, :], pos_embed_sampled], 1 + ) + x += pos_embed_sampled + + # apply Transformer blocks + B, N, C = x.shape + thw = [T, H, W] + for _, blk in enumerate(self.blocks): + x, thw = blk(x, thw) + x = self.norm(x) + + return x, mask, ids_restore, thw + + def _mae_forward_decoder(self, x, ids_restore, mask, thw): + # embed tokens + x = self.decoder_embed(x) + T, H, W = self.T, self.H, self.W + B, N, C = x.shape + + s = 1 if self.cls_embed_on else 0 + + # append mask tokens to sequence + mask_tokens = self.mask_token.repeat( + B, T * H * W + s - x.shape[1], 1 + ) # + s: no cls token + x_ = torch.cat([x[:, s:, :], mask_tokens], dim=1) # no cls token + if self.cfg.MASK.PER_FRAME_MASKING: + x_ = x_.view([B * T, H * W, C]) + x_ = torch.gather( + x_, dim=1, index=ids_restore.unsqueeze(-1).repeat(1, 1, x_.shape[2]) + ) # unshuffle + if self.cfg.MASK.PER_FRAME_MASKING: + x_ = x_.view([B, T * H * W, C]) + x = torch.cat([x[:, :s, :], x_], dim=1) # append cls token + + if self.sep_pos_embed_decoder: + pos_embed = self.dec_pos_embed_spatial.repeat( + 1, self.patch_dims[0], 1 + ) + torch.repeat_interleave( + self.dec_pos_embed_temporal, + self.patch_dims[1] * self.patch_dims[2], + dim=1, + ) + pos_embed = pos_embed.expand(x.shape[0], -1, -1) + if self.cls_embed_on: + pos_embed = torch.cat( + [ + self.dec_pos_embed_class.expand( + pos_embed.shape[0], -1, -1 + ), + pos_embed, + ], + 1, + ) + x += pos_embed + else: + # add pos embed + x = x + self.decoder_pos_embed + + pixel_outputs = self.pred_head( + [x], + [mask.to(torch.bool)], + return_all=self.cfg.VIS_MASK.ENABLE, + thw=thw, + ) + + return pixel_outputs + + def _mae_forward(self, imgs, mask_ratio=0.75, mask=None): + latent, mask, ids_restore, thw = self._mae_forward_encoder( + imgs, mask_ratio, mask + ) + pred = self._mae_forward_decoder(latent, ids_restore, mask, thw) + labels = [] + if self.pred_pixel_wt: + if self.use_2d_patch: + labels += self._get_pixel_label_2d( + imgs.detach(), + [mask.to(torch.bool)], + norm=self.cfg.MASK.NORM_PRED_PIXEL, + ) + else: + labels += self._get_pixel_label_3d( + imgs.detach(), + [mask.to(torch.bool)], + time_stride_loss=self.cfg.MASK.TIME_STRIDE_LOSS, + norm=self.cfg.MASK.NORM_PRED_PIXEL, + ) + if self.pred_hog_wt: + if self.use_2d_patch: + labels += self._get_hog_label_2d( + imgs.detach(), [mask.to(torch.bool)] + ) + else: + labels += self._get_hog_label_3d( + imgs.detach(), [mask.to(torch.bool)] + ) + + self.counter += 1 + if self.cfg.VIS_MASK.ENABLE: + return self._mae_visualize(imgs, pred, mask) + return pred, labels + + def _mae_visualize(self, imgs, pred, mask): + N, T, H, W, p, u, t, h, w = self.patch_info + pred = pred[0] + if self.cfg.MASK.TIME_STRIDE_LOSS: + im_viz = imgs[:, :, :: self.cfg.MVIT.PATCH_STRIDE[0], :, :] + else: + im_viz = imgs + reconstruct = self._unpatchify( + pred * mask.reshape(N, t * h * w, 1) + + self._patchify( + im_viz, time_stride_loss=self.cfg.MASK.TIME_STRIDE_LOSS + ) + * (1 - mask.reshape(N, t * h * w, 1)) + ) + masked = self._unpatchify( + self._patchify( + im_viz, time_stride_loss=self.cfg.MASK.TIME_STRIDE_LOSS + ) + * (1 - mask.reshape(N, t * h * w, 1)) + ) + + comparison = torch.stack( + [im_viz, masked, reconstruct], + dim=1, + ).permute([0, 1, 3, 2, 4, 5]) + pfx = self.cfg.TEST.CHECKPOINT_FILE_PATH + mr = self.cfg.AUG.MASK_RATIO + for i in range(comparison.shape[0]): + misc.plot_input_normed( + comparison[i].cpu(), + bboxes=(), + texts=(), + path=self.cfg.OUTPUT_DIR + + "/vis_mask/vid/{}vis_video_in_mask_out_mr{}/vis_{}_{}.mp4".format( + pfx[pfx.rfind("/") + 1 : -5], mr, self.counter, i + ), + folder_path=self.cfg.OUTPUT_DIR + + "/vis_mask/vid/{}vis_video_in_mask_out_mr{}".format( + pfx[pfx.rfind("/") + 1 : -5], mr + ), + make_grids=True, + output_video=True, + ) + return pred[0] + + def _maskfeat_forward(self, x, mask, return_all=False): + x_embed, x_shape = self.patch_embed(x) + if self.cfg.MASK.MAE_RND_MASK: + _, mask, ids_restore, ids_keep = self._mae_random_masking( + x_embed, self.cfg.AUG.MASK_RATIO, None + ) + output_masks = [mask.to(torch.bool)] + else: + # take masks and labels from loader + float_mask = mask.type_as(x) + output_masks = self._get_multiscale_mask(float_mask) + labels = [] + if self.pred_pixel_wt: + if self.use_2d_patch: + labels += self._get_pixel_label_2d( + x.detach(), output_masks, norm=self.cfg.MASK.NORM_PRED_PIXEL + ) + else: + labels += self._get_pixel_label_3d( + x.detach(), output_masks, norm=self.cfg.MASK.NORM_PRED_PIXEL + ) + if self.pred_hog_wt: + if self.use_2d_patch: + labels += self._get_hog_label_2d(x.detach(), output_masks) + else: + labels += self._get_hog_label_3d(x.detach(), output_masks) + + x = x_embed + T, H, W = self.T, self.H, self.W + B, N, C = x.shape + + # switch input tokens by mask_token + mask_tokens = self.mask_token.expand(B, N, -1) + if self.cfg.MASK.MAE_RND_MASK: + float_mask = mask.unsqueeze(-1) + else: + if self.use_2d_patch: + float_mask = F.interpolate( + float_mask.unsqueeze(0), size=(H, W) + )[0] + else: + float_mask = F.interpolate(float_mask, size=(H, W)) + float_mask = float_mask.flatten(1).unsqueeze(-1) + x = x * (1 - float_mask) + mask_tokens * float_mask + + if self.cls_embed_on: + cls_tokens = self.cls_token.expand(B, -1, -1) + x = torch.cat((cls_tokens, x), dim=1) + + if self.use_abs_pos: + if self.sep_pos_embed: + pos_embed = self.pos_embed_spatial.repeat( + 1, self.patch_dims[0], 1 + ) + torch.repeat_interleave( + self.pos_embed_temporal, + self.patch_dims[1] * self.patch_dims[2], + dim=1, + ) + if self.cls_embed_on: + pos_embed = torch.cat([self.pos_embed_class, pos_embed], 1) + x = x + pos_embed + else: + x = x + self.pos_embed + + if self.drop_rate: + x = self.pos_drop(x) + + if self.norm_stem: + x = self.norm_stem(x) + + thw = [T, H, W] + block_outputs = [] + for idx, blk in enumerate(self.blocks): + x, thw = blk(x, thw) + if idx in self.pretrain_depth: + block_outputs.append(x) + + model_outputs = [] + if self.pred_pixel_wt: + pixel_outputs = self.pred_head( + block_outputs, + output_masks, + return_all=return_all, + thw=thw, + ) + model_outputs += pixel_outputs + if self.pred_hog_wt: + hog_outputs = self.pred_head( + block_outputs, + output_masks, + return_all=return_all, + thw=thw, + ) + model_outputs += hog_outputs + + return model_outputs, labels + + def forward(self, x, return_all=False): + if len(x) > 1: + x, meta, mask = x + else: + x, mask = x[0], None + + if self.cfg.MASK.MAE_ON: + return self._mae_forward( + x, mask_ratio=self.cfg.AUG.MASK_RATIO, mask=mask + ) + else: + return self._maskfeat_forward(x, mask, return_all) diff --git a/slowfast/models/operators.py b/slowfast/models/operators.py index 1e4a0433e..4ca614503 100644 --- a/slowfast/models/operators.py +++ b/slowfast/models/operators.py @@ -3,7 +3,13 @@ """Custom operators.""" +import math +import torch import torch.nn as nn +import torch.nn.functional as F +import torchvision + +from slowfast.models.utils import get_gkern from pytorchvideo.layers.swish import Swish @@ -55,3 +61,62 @@ def forward(self, x): for module in self.children(): x = module(x) return x_in * x + + +class HOGLayerC(nn.Module): + def __init__(self, nbins=9, pool=7, gaussian_window=0): + super(HOGLayerC, self).__init__() + self.nbins = nbins + self.pool = pool + self.pi = math.pi + weight_x = torch.FloatTensor([[1, 0, -1], [2, 0, -2], [1, 0, -1]]) + weight_x = weight_x.view(1, 1, 3, 3).repeat(3, 1, 1, 1) + weight_y = weight_x.transpose(2, 3) + self.register_buffer("weight_x", weight_x) + self.register_buffer("weight_y", weight_y) + + self.gaussian_window = gaussian_window + if gaussian_window: + gkern = get_gkern(gaussian_window, gaussian_window // 2) + self.register_buffer("gkern", gkern) + + @torch.no_grad() + def forward(self, x): + # input is RGB image with shape [B 3 H W] + x = F.pad(x, pad=(1, 1, 1, 1), mode="reflect") + gx_rgb = F.conv2d( + x, self.weight_x, bias=None, stride=1, padding=0, groups=3 + ) + gy_rgb = F.conv2d( + x, self.weight_y, bias=None, stride=1, padding=0, groups=3 + ) + norm_rgb = torch.stack([gx_rgb, gy_rgb], dim=-1).norm(dim=-1) + phase = torch.atan2(gx_rgb, gy_rgb) + phase = phase / self.pi * self.nbins # [-9, 9] + + b, c, h, w = norm_rgb.shape + out = torch.zeros( + (b, c, self.nbins, h, w), dtype=torch.float, device=x.device + ) + phase = phase.view(b, c, 1, h, w) + norm_rgb = norm_rgb.view(b, c, 1, h, w) + if self.gaussian_window: + if h != self.gaussian_window: + assert h % self.gaussian_window == 0, "h {} gw {}".format( + h, self.gaussian_window + ) + repeat_rate = h // self.gaussian_window + temp_gkern = self.gkern.repeat([repeat_rate, repeat_rate]) + else: + temp_gkern = self.gkern + norm_rgb *= temp_gkern + + out.scatter_add_(2, phase.floor().long() % self.nbins, norm_rgb) + + out = out.unfold(3, self.pool, self.pool) + out = out.unfold(4, self.pool, self.pool) + out = out.sum(dim=[-1, -2]) + + out = torch.nn.functional.normalize(out, p=2, dim=2) + + return out # B 3 nbins H W diff --git a/slowfast/models/optimizer.py b/slowfast/models/optimizer.py index 0ca5c4721..ceb2a32fb 100644 --- a/slowfast/models/optimizer.py +++ b/slowfast/models/optimizer.py @@ -23,74 +23,87 @@ def construct_optimizer(model, cfg): cfg (config): configs of hyper-parameters of SGD or ADAM, includes base learning rate, momentum, weight_decay, dampening, and etc. """ - bn_parameters = [] - non_bn_parameters = [] - zero_parameters = [] - no_grad_parameters = [] - skip = {} - if cfg.NUM_GPUS > 1: - if hasattr(model.module, "no_weight_decay"): - skip = model.module.no_weight_decay() - skip = {"module." + v for v in skip} - else: - if hasattr(model, "no_weight_decay"): - skip = model.no_weight_decay() - - for name, m in model.named_modules(): - is_bn = isinstance(m, torch.nn.modules.batchnorm._NormBase) - for p in m.parameters(recurse=False): - if not p.requires_grad: - no_grad_parameters.append(p) - elif is_bn: - bn_parameters.append(p) - elif name in skip: - zero_parameters.append(p) - elif cfg.SOLVER.ZERO_WD_1D_PARAM and ( - len(p.shape) == 1 or name.endswith(".bias") - ): - zero_parameters.append(p) - else: - non_bn_parameters.append(p) - - optim_params = [ - { - "params": bn_parameters, - "weight_decay": cfg.BN.WEIGHT_DECAY, - "apply_LARS": False, - }, - { - "params": non_bn_parameters, - "weight_decay": cfg.SOLVER.WEIGHT_DECAY, - "apply_LARS": cfg.SOLVER.LARS_ON, - }, - { - "params": zero_parameters, - "weight_decay": 0.0, - "apply_LARS": cfg.SOLVER.LARS_ON, - }, - ] - optim_params = [x for x in optim_params if len(x["params"])] - - # Check all parameters will be passed into optimizer. - assert len(list(model.parameters())) == len(non_bn_parameters) + len( - bn_parameters - ) + len(zero_parameters) + len( - no_grad_parameters - ), "parameter size does not match: {} + {} + {} + {} != {}".format( - len(non_bn_parameters), - len(bn_parameters), - len(zero_parameters), - len(no_grad_parameters), - len(list(model.parameters())), - ) - print( - "bn {}, non bn {}, zero {} no grad {}".format( - len(bn_parameters), + if cfg.SOLVER.LAYER_DECAY > 0.0 and cfg.SOLVER.LAYER_DECAY < 1.0: + optim_params = get_param_groups(model, cfg) + elif cfg.SOLVER.LAYER_DECAY == 1.0: + bn_parameters = [] + non_bn_parameters = [] + zero_parameters = [] + no_grad_parameters = [] + skip = {} + + if cfg.NUM_GPUS > 1: + if hasattr(model.module, "no_weight_decay"): + skip = model.module.no_weight_decay() + else: + if hasattr(model, "no_weight_decay"): + skip = model.no_weight_decay() + + for name_m, m in model.named_modules(): + is_bn = isinstance(m, torch.nn.modules.batchnorm._NormBase) + for name_p, p in m.named_parameters(recurse=False): + name = "{}.{}".format(name_m, name_p).strip(".") + if not p.requires_grad: + no_grad_parameters.append(p) + elif is_bn: + bn_parameters.append(p) + elif any(k in name for k in skip): + zero_parameters.append(p) + elif cfg.SOLVER.ZERO_WD_1D_PARAM and ( + len(p.shape) == 1 or name.endswith(".bias") + ): + zero_parameters.append(p) + else: + non_bn_parameters.append(p) + + optim_params = [ + { + "params": bn_parameters, + "weight_decay": cfg.BN.WEIGHT_DECAY, + "layer_decay": 1.0, + "apply_LARS": False, + }, + { + "params": non_bn_parameters, + "weight_decay": cfg.SOLVER.WEIGHT_DECAY, + "layer_decay": 1.0, + "apply_LARS": cfg.SOLVER.LARS_ON, + }, + { + "params": zero_parameters, + "weight_decay": 0.0, + "layer_decay": 1.0, + "apply_LARS": cfg.SOLVER.LARS_ON, + }, + ] + optim_params = [x for x in optim_params if len(x["params"])] + + # Check all parameters will be passed into optimizer. + assert len(list(model.parameters())) == len(non_bn_parameters) + len( + bn_parameters + ) + len(zero_parameters) + len( + no_grad_parameters + ), "parameter size does not match: {} + {} + {} + {} != {}".format( len(non_bn_parameters), + len(bn_parameters), len(zero_parameters), len(no_grad_parameters), + len(list(model.parameters())), + ) + print( + "bn {}, non bn {}, zero {}, no grad {}".format( + len(bn_parameters), + len(non_bn_parameters), + len(zero_parameters), + len(no_grad_parameters), + ) + ) + else: + raise ValueError( + "Layer decay should be in (0, 1], but is {}".format( + cfg.SOLVER.LAYER_DECAY + ) ) - ) if cfg.SOLVER.OPTIMIZING_METHOD == "sgd": optimizer = torch.optim.SGD( @@ -105,13 +118,14 @@ def construct_optimizer(model, cfg): optimizer = torch.optim.Adam( optim_params, lr=cfg.SOLVER.BASE_LR, - betas=(0.9, 0.999), + betas=cfg.SOLVER.BETAS, weight_decay=cfg.SOLVER.WEIGHT_DECAY, ) elif cfg.SOLVER.OPTIMIZING_METHOD == "adamw": optimizer = torch.optim.AdamW( optim_params, lr=cfg.SOLVER.BASE_LR, + betas=cfg.SOLVER.BETAS, eps=1e-08, weight_decay=cfg.SOLVER.WEIGHT_DECAY, ) @@ -119,6 +133,7 @@ def construct_optimizer(model, cfg): optimizer = torch.optim._multi_tensor.AdamW( optim_params, lr=cfg.SOLVER.BASE_LR, + betas=cfg.SOLVER.BETAS, eps=1e-08, weight_decay=cfg.SOLVER.WEIGHT_DECAY, ) @@ -133,6 +148,102 @@ def construct_optimizer(model, cfg): return optimizer +def get_param_groups(model, cfg): + def _get_layer_decay(name): + layer_id = None + if name in ("cls_token", "mask_token"): + layer_id = 0 + elif name.startswith("pos_embed"): + layer_id = 0 + elif name.startswith("patch_embed"): + layer_id = 0 + elif name.startswith("blocks"): + layer_id = int(name.split(".")[1]) + 1 + else: + layer_id = cfg.MVIT.DEPTH + 1 + layer_decay = cfg.SOLVER.LAYER_DECAY ** (cfg.MVIT.DEPTH + 1 - layer_id) + return layer_id, layer_decay + + for m in model.modules(): + assert not isinstance( + m, torch.nn.modules.batchnorm._NormBase + ), "BN is not supported with layer decay" + + non_bn_parameters_count = 0 + zero_parameters_count = 0 + no_grad_parameters_count = 0 + parameter_group_names = {} + parameter_group_vars = {} + + skip = {} + if cfg.NUM_GPUS > 1: + if hasattr(model.module, "no_weight_decay"): + skip = model.module.no_weight_decay() + # skip = {"module." + v for v in skip} + else: + if hasattr(model, "no_weight_decay"): + skip = model.no_weight_decay() + + for name, p in model.named_parameters(): + if not p.requires_grad: + group_name = "no_grad" + no_grad_parameters_count += 1 + continue + name = name[len("module.") :] if name.startswith("module.") else name + if name in skip or ( + (len(p.shape) == 1 or name.endswith(".bias")) + and cfg.SOLVER.ZERO_WD_1D_PARAM + ): + layer_id, layer_decay = _get_layer_decay(name) + group_name = "layer_%d_%s" % (layer_id, "zero") + weight_decay = 0.0 + zero_parameters_count += 1 + else: + layer_id, layer_decay = _get_layer_decay(name) + group_name = "layer_%d_%s" % (layer_id, "non_bn") + weight_decay = cfg.SOLVER.WEIGHT_DECAY + non_bn_parameters_count += 1 + + if group_name not in parameter_group_names: + parameter_group_names[group_name] = { + "weight_decay": weight_decay, + "params": [], + "layer_decay": layer_decay, + } + parameter_group_vars[group_name] = { + "weight_decay": weight_decay, + "params": [], + "layer_decay": layer_decay, + } + parameter_group_names[group_name]["params"].append(name) + parameter_group_vars[group_name]["params"].append(p) + + # print("Param groups = %s" % json.dumps(parameter_group_names, indent=2)) + optim_params = list(parameter_group_vars.values()) + + # Check all parameters will be passed into optimizer. + assert ( + len(list(model.parameters())) + == non_bn_parameters_count + + zero_parameters_count + + no_grad_parameters_count + ), "parameter size does not match: {} + {} + {} != {}".format( + non_bn_parameters_count, + zero_parameters_count, + no_grad_parameters_count, + len(list(model.parameters())), + ) + print( + "non bn {}, zero {}, no grad {}".format( + non_bn_parameters_count, + zero_parameters_count, + no_grad_parameters_count, + ) + ) + + return optim_params + + def get_epoch_lr(cur_epoch, cfg): """ Retrieves the lr for the given epoch (as specified by the lr policy). @@ -152,7 +263,7 @@ def set_lr(optimizer, new_lr): new_lr (float): the new learning rate to set. """ for param_group in optimizer.param_groups: - param_group["lr"] = new_lr + param_group["lr"] = new_lr * param_group["layer_decay"] class LARS(object): @@ -257,3 +368,28 @@ def step(self): # return weight decay control to optimizer for i, group in enumerate(self.optim.param_groups): group["weight_decay"] = weight_decays[i] + + +def get_grad_norm_(parameters, norm_type=2.0): + if isinstance(parameters, torch.Tensor): + parameters = [parameters] + parameters = [p for p in parameters if p.grad is not None] + norm_type = float(norm_type) + if len(parameters) == 0: + return torch.tensor(0.0) + device = parameters[0].grad.device + if norm_type == "inf": + total_norm = max( + p.grad.detach().abs().max().to(device) for p in parameters + ) + else: + total_norm = torch.norm( + torch.stack( + [ + torch.norm(p.grad.detach(), norm_type).to(device) + for p in parameters + ] + ), + norm_type, + ) + return total_norm diff --git a/slowfast/models/stem_helper.py b/slowfast/models/stem_helper.py index 889961317..65515816d 100644 --- a/slowfast/models/stem_helper.py +++ b/slowfast/models/stem_helper.py @@ -3,6 +3,7 @@ """ResNe(X)t 3D stem helper.""" +import torch import torch.nn as nn @@ -316,7 +317,9 @@ def __init__( padding=padding, ) - def forward(self, x): + def forward(self, x, keep_spatial=False): x = self.proj(x) + if keep_spatial: + return x, x.shape # B C (T) H W -> B (T)HW C return x.flatten(2).transpose(1, 2), x.shape diff --git a/slowfast/models/utils.py b/slowfast/models/utils.py index 381ba98fc..7e06c491e 100644 --- a/slowfast/models/utils.py +++ b/slowfast/models/utils.py @@ -1,5 +1,8 @@ # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +import numpy as np +import torch + import slowfast.utils.logging as logging logger = logging.get_logger(__name__) @@ -27,3 +30,196 @@ def validate_checkpoint_wrapper_import(checkpoint_wrapper): """ if checkpoint_wrapper is None: raise ImportError("Please install fairscale.") + + +def get_gkern(kernlen, std): + """Returns a 2D Gaussian kernel array.""" + + def _gaussian_fn(kernlen, std): + n = torch.arange(0, kernlen).float() + n -= n.mean() + n /= std + w = torch.exp(-0.5 * n**2) + return w + + gkern1d = _gaussian_fn(kernlen, std) + gkern2d = torch.outer(gkern1d, gkern1d) + return gkern2d / gkern2d.sum() + + +# -------------------------------------------------------- +# 2D sine-cosine position embedding +# References: +# Transformer: https://github.com/tensorflow/models/blob/master/official/nlp/transformer/model_utils.py +# MoCo v3: https://github.com/facebookresearch/moco-v3 +# -------------------------------------------------------- +def get_3d_sincos_pos_embed(embed_dim, grid_size, t_size, cls_token=False): + """ + grid_size: int of the grid height and width + t_size: int of the temporal size + return: + pos_embed: [t_size*grid_size*grid_size, embed_dim] or [1+t_size*grid_size*grid_size, embed_dim] (w/ or w/o cls_token) + """ + assert embed_dim % 4 == 0 + embed_dim_spatial = embed_dim // 4 * 3 + embed_dim_temporal = embed_dim // 4 + + # spatial + grid_h = np.arange(grid_size, dtype=np.float32) + grid_w = np.arange(grid_size, dtype=np.float32) + grid = np.meshgrid(grid_w, grid_h) # here w goes first + grid = np.stack(grid, axis=0) + + grid = grid.reshape([2, 1, grid_size, grid_size]) + pos_embed_spatial = get_2d_sincos_pos_embed_from_grid( + embed_dim_spatial, grid + ) + + # temporal + grid_t = np.arange(t_size, dtype=np.float32) + pos_embed_temporal = get_1d_sincos_pos_embed_from_grid( + embed_dim_temporal, grid_t + ) + + # concate: [T, H, W] order + pos_embed_temporal = pos_embed_temporal[:, np.newaxis, :] + pos_embed_temporal = np.repeat( + pos_embed_temporal, grid_size**2, axis=1 + ) # [T, H*W, D // 4] + pos_embed_spatial = pos_embed_spatial[np.newaxis, :, :] + pos_embed_spatial = np.repeat( + pos_embed_spatial, t_size, axis=0 + ) # [T, H*W, D // 4 * 3] + + pos_embed = np.concatenate([pos_embed_temporal, pos_embed_spatial], axis=-1) + pos_embed = pos_embed.reshape([-1, embed_dim]) # [T*H*W, D] + + if cls_token: + pos_embed = np.concatenate( + [np.zeros([1, embed_dim]), pos_embed], axis=0 + ) + return pos_embed + + +def get_2d_sincos_pos_embed(embed_dim, grid_size, cls_token=False): + """ + grid_size: int of the grid height and width + return: + pos_embed: [grid_size*grid_size, embed_dim] or [1+grid_size*grid_size, embed_dim] (w/ or w/o cls_token) + """ + grid_h = np.arange(grid_size, dtype=np.float32) + grid_w = np.arange(grid_size, dtype=np.float32) + grid = np.meshgrid(grid_w, grid_h) # here w goes first + grid = np.stack(grid, axis=0) + + grid = grid.reshape([2, 1, grid_size, grid_size]) + pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid) + if cls_token: + pos_embed = np.concatenate( + [np.zeros([1, embed_dim]), pos_embed], axis=0 + ) + return pos_embed + + +def get_2d_sincos_pos_embed_from_grid(embed_dim, grid): + assert embed_dim % 2 == 0 + + # use half of dimensions to encode grid_h + emb_h = get_1d_sincos_pos_embed_from_grid( + embed_dim // 2, grid[0] + ) # (H*W, D/2) + emb_w = get_1d_sincos_pos_embed_from_grid( + embed_dim // 2, grid[1] + ) # (H*W, D/2) + + emb = np.concatenate([emb_h, emb_w], axis=1) # (H*W, D) + return emb + + +def get_1d_sincos_pos_embed_from_grid(embed_dim, pos): + """ + embed_dim: output dimension for each position + pos: a list of positions to be encoded: size (M,) + out: (M, D) + """ + assert embed_dim % 2 == 0 + omega = np.arange(embed_dim // 2, dtype=np.float) + omega /= embed_dim / 2.0 + omega = 1.0 / 10000**omega # (D/2,) + + pos = pos.reshape(-1) # (M,) + out = np.einsum("m,d->md", pos, omega) # (M, D/2), outer product + + emb_sin = np.sin(out) # (M, D/2) + emb_cos = np.cos(out) # (M, D/2) + + emb = np.concatenate([emb_sin, emb_cos], axis=1) # (M, D) + return emb + + +# -------------------------------------------------------- +# Interpolate position embeddings for high-resolution +# References: +# DeiT: https://github.com/facebookresearch/deit +# -------------------------------------------------------- +def interpolate_pos_embed(model, checkpoint_model): + if "pos_embed" in checkpoint_model: + pos_embed_checkpoint = checkpoint_model["pos_embed"] + embedding_size = pos_embed_checkpoint.shape[-1] + num_patches = model.patch_embed.num_patches + num_extra_tokens = model.pos_embed.shape[-2] - num_patches + # height (== width) for the checkpoint position embedding + orig_size = int( + (pos_embed_checkpoint.shape[-2] - num_extra_tokens) ** 0.5 + ) + # height (== width) for the new position embedding + new_size = int(num_patches**0.5) + # class_token and dist_token are kept unchanged + if orig_size != new_size: + print( + "Position interpolate from %dx%d to %dx%d" + % (orig_size, orig_size, new_size, new_size) + ) + extra_tokens = pos_embed_checkpoint[:, :num_extra_tokens] + # only the position tokens are interpolated + pos_tokens = pos_embed_checkpoint[:, num_extra_tokens:] + pos_tokens = pos_tokens.reshape( + -1, orig_size, orig_size, embedding_size + ).permute(0, 3, 1, 2) + pos_tokens = torch.nn.functional.interpolate( + pos_tokens, + size=(new_size, new_size), + mode="bicubic", + align_corners=False, + ) + pos_tokens = pos_tokens.permute(0, 2, 3, 1).flatten(1, 2) + new_pos_embed = torch.cat((extra_tokens, pos_tokens), dim=1) + checkpoint_model["pos_embed"] = new_pos_embed + + +def calc_mvit_feature_geometry(cfg): + feat_size = [ + [ + cfg.DATA.NUM_FRAMES // cfg.MVIT.PATCH_STRIDE[0] + if len(cfg.MVIT.PATCH_STRIDE) > 2 + else 1, + cfg.DATA.TRAIN_CROP_SIZE // cfg.MVIT.PATCH_STRIDE[-2], + cfg.DATA.TRAIN_CROP_SIZE // cfg.MVIT.PATCH_STRIDE[-1], + ] + for i in range(cfg.MVIT.DEPTH) + ] + feat_stride = [ + [ + cfg.MVIT.PATCH_STRIDE[0] if len(cfg.MVIT.PATCH_STRIDE) > 2 else 1, + cfg.MVIT.PATCH_STRIDE[-2], + cfg.MVIT.PATCH_STRIDE[-1], + ] + for i in range(cfg.MVIT.DEPTH) + ] + for _, x in enumerate(cfg.MVIT.POOL_Q_STRIDE): + for i in range(cfg.MVIT.DEPTH): + if i >= x[0]: + for j in range(len(feat_size[i])): + feat_size[i][j] = feat_size[i][j] // x[j + 1] + feat_stride[i][j] = feat_stride[i][j] * x[j + 1] + return feat_size, feat_stride diff --git a/slowfast/models/video_model_builder.py b/slowfast/models/video_model_builder.py index 44b20db5e..3f4a85fbd 100644 --- a/slowfast/models/video_model_builder.py +++ b/slowfast/models/video_model_builder.py @@ -10,15 +10,18 @@ import torch.nn.functional as F from torch.nn.init import trunc_normal_ +import slowfast.utils.logging as logging import slowfast.utils.weight_init_helper as init_helper from slowfast.models.attention import MultiScaleBlock from slowfast.models.batchnorm_helper import get_norm from slowfast.models.utils import ( + calc_mvit_feature_geometry, + get_3d_sincos_pos_embed, round_width, validate_checkpoint_wrapper_import, ) -from . import head_helper, resnet_helper, stem_helper +from . import head_helper, operators, resnet_helper, stem_helper # noqa from .build import MODEL_REGISTRY try: @@ -26,6 +29,7 @@ except ImportError: checkpoint_wrapper = None +logger = logging.get_logger(__name__) # Number of blocks for different stages given the model depth. _MODEL_STAGE_DEPTH = {18: (2, 2, 2, 2), 50: (3, 4, 6, 3), 101: (3, 4, 23, 3)} @@ -822,10 +826,14 @@ def __init__(self, cfg): spatial_size = cfg.DATA.TRAIN_CROP_SIZE temporal_size = cfg.DATA.NUM_FRAMES in_chans = cfg.DATA.INPUT_CHANNEL_NUM[0] - use_2d_patch = cfg.MVIT.PATCH_2D + self.use_2d_patch = cfg.MVIT.PATCH_2D + self.enable_detection = cfg.DETECTION.ENABLE self.patch_stride = cfg.MVIT.PATCH_STRIDE - if use_2d_patch: + if self.use_2d_patch: self.patch_stride = [1] + self.patch_stride + self.T = cfg.DATA.NUM_FRAMES // self.patch_stride[0] + self.H = cfg.DATA.TRAIN_CROP_SIZE // self.patch_stride[1] + self.W = cfg.DATA.TRAIN_CROP_SIZE // self.patch_stride[2] # Prepare output. num_classes = cfg.MODEL.NUM_CLASSES embed_dim = cfg.MVIT.EMBED_DIM @@ -836,10 +844,14 @@ def __init__(self, cfg): self.drop_rate = cfg.MVIT.DROPOUT_RATE depth = cfg.MVIT.DEPTH drop_path_rate = cfg.MVIT.DROPPATH_RATE + layer_scale_init_value = cfg.MVIT.LAYER_SCALE_INIT_VALUE + head_init_scale = cfg.MVIT.HEAD_INIT_SCALE mode = cfg.MVIT.MODE self.cls_embed_on = cfg.MVIT.CLS_EMBED_ON + self.use_mean_pooling = cfg.MVIT.USE_MEAN_POOLING # Params for positional embedding self.use_abs_pos = cfg.MVIT.USE_ABS_POS + self.use_fixed_sincos_pos = cfg.MVIT.USE_FIXED_SINCOS_POS self.sep_pos_embed = cfg.MVIT.SEP_POS_EMBED self.rel_pos_spatial = cfg.MVIT.REL_POS_SPATIAL self.rel_pos_temporal = cfg.MVIT.REL_POS_TEMPORAL @@ -848,17 +860,17 @@ def __init__(self, cfg): else: raise NotImplementedError("Only supports layernorm.") self.num_classes = num_classes - patch_embed = stem_helper.PatchEmbed( + self.patch_embed = stem_helper.PatchEmbed( dim_in=in_chans, dim_out=embed_dim, kernel=cfg.MVIT.PATCH_KERNEL, stride=cfg.MVIT.PATCH_STRIDE, padding=cfg.MVIT.PATCH_PADDING, - conv_2d=use_2d_patch, + conv_2d=self.use_2d_patch, ) + if cfg.MODEL.ACT_CHECKPOINT: - patch_embed = checkpoint_wrapper(patch_embed) - self.patch_embed = patch_embed + self.patch_embed = checkpoint_wrapper(self.patch_embed) self.input_dims = [temporal_size, spatial_size, spatial_size] assert self.input_dims[1] == self.input_dims[2] self.patch_dims = [ @@ -893,7 +905,12 @@ def __init__(self, cfg): ) else: self.pos_embed = nn.Parameter( - torch.zeros(1, pos_embed_dim, embed_dim) + torch.zeros( + 1, + pos_embed_dim, + embed_dim, + ), + requires_grad=not self.use_fixed_sincos_pos, ) if self.drop_rate > 0.0: @@ -978,6 +995,7 @@ def __init__(self, cfg): qkv_bias=qkv_bias, drop_rate=self.drop_rate, drop_path=dpr[i], + layer_scale_init_value=layer_scale_init_value, norm_layer=norm_layer, kernel_q=pool_q[i] if len(pool_q) > i else [], kernel_kv=pool_kv[i] if len(pool_kv) > i else [], @@ -996,22 +1014,35 @@ def __init__(self, cfg): if cfg.MODEL.ACT_CHECKPOINT: attention_block = checkpoint_wrapper(attention_block) self.blocks.append(attention_block) - if len(stride_q[i]) > 0: input_size = [ size // stride for size, stride in zip(input_size, stride_q[i]) ] + embed_dim = dim_out + self.norm = norm_layer(embed_dim) - self.head = head_helper.TransformerBasicHead( - embed_dim, - num_classes, - dropout_rate=cfg.MODEL.DROPOUT_RATE, - act_func=cfg.MODEL.HEAD_ACT, - cfg=cfg, - ) + if self.enable_detection: + self.head = head_helper.ResNetRoIHead( + dim_in=[embed_dim], + num_classes=num_classes, + pool_size=[[temporal_size // self.patch_stride[0], 1, 1]], + resolution=[[cfg.DETECTION.ROI_XFORM_RESOLUTION] * 2], + scale_factor=[cfg.DETECTION.SPATIAL_SCALE_FACTOR], + dropout_rate=cfg.MODEL.DROPOUT_RATE, + act_func=cfg.MODEL.HEAD_ACT, + aligned=cfg.DETECTION.ALIGNED, + ) + else: + self.head = head_helper.TransformerBasicHead( + embed_dim, + num_classes, + dropout_rate=cfg.MODEL.DROPOUT_RATE, + act_func=cfg.MODEL.HEAD_ACT, + cfg=cfg, + ) if self.use_abs_pos: if self.sep_pos_embed: trunc_normal_(self.pos_embed_spatial, std=0.02) @@ -1020,12 +1051,28 @@ def __init__(self, cfg): trunc_normal_(self.pos_embed_class, std=0.02) else: trunc_normal_(self.pos_embed, std=0.02) + if self.use_fixed_sincos_pos: + pos_embed = get_3d_sincos_pos_embed( + self.pos_embed.shape[-1], + self.H, + self.T, + cls_token=self.cls_embed_on, + ) + self.pos_embed.data.copy_( + torch.from_numpy(pos_embed).float().unsqueeze(0) + ) + if self.cls_embed_on: trunc_normal_(self.cls_token, std=0.02) self.apply(self._init_weights) + self.head.projection.weight.data.mul_(head_init_scale) + self.head.projection.bias.data.mul_(head_init_scale) + + self.feat_size, self.feat_stride = calc_mvit_feature_geometry(cfg) + def _init_weights(self, m): - if isinstance(m, nn.Linear): + if isinstance(m, (nn.Linear, nn.Conv2d, nn.Conv3d)): nn.init.trunc_normal_(m.weight, std=0.02) if isinstance(m, nn.Linear) and m.bias is not None: nn.init.constant_(m.bias, 0) @@ -1081,18 +1128,26 @@ def _get_pos_embed(self, pos_embed, bcthw): return pos_embed - def forward(self, x): + def forward(self, x, bboxes=None, return_attn=False): x = x[0] x, bcthw = self.patch_embed(x) - - T = self.cfg.DATA.NUM_FRAMES // self.patch_stride[0] - H, W = bcthw[-2], bcthw[-1] + bcthw = list(bcthw) + if len(bcthw) == 4: # Fix bcthw in case of 4D tensor + bcthw.insert(2, torch.tensor(self.T)) + T, H, W = bcthw[-3], bcthw[-2], bcthw[-1] + assert len(bcthw) == 5 and (T, H, W) == (self.T, self.H, self.W), bcthw B, N, C = x.shape + s = 1 if self.cls_embed_on else 0 + if self.use_fixed_sincos_pos: + x += self.pos_embed[:, s:, :] # s: on/off cls token + if self.cls_embed_on: cls_tokens = self.cls_token.expand( B, -1, -1 ) # stole cls_tokens impl from Phil Wang, thanks + if self.use_fixed_sincos_pos: + cls_tokens = cls_tokens + self.pos_embed[:, :s, :] x = torch.cat((cls_tokens, x), dim=1) if self.use_abs_pos: @@ -1106,11 +1161,9 @@ def forward(self, x): ) if self.cls_embed_on: pos_embed = torch.cat([self.pos_embed_class, pos_embed], 1) - pos_embed = self._get_pos_embed(pos_embed, bcthw) - x = x + pos_embed + x += self._get_pos_embed(pos_embed, bcthw) else: - pos_embed = self._get_pos_embed(self.pos_embed, bcthw) - x = x + pos_embed + x += self._get_pos_embed(self.pos_embed, bcthw) if self.drop_rate: x = self.pos_drop(x) @@ -1119,14 +1172,30 @@ def forward(self, x): x = self.norm_stem(x) thw = [T, H, W] - for blk in self.blocks: + for i, blk in enumerate(self.blocks): x, thw = blk(x, thw) - x = self.norm(x) - if self.cls_embed_on: - x = x[:, 0] + if self.enable_detection: + x = self.norm(x) + if self.cls_embed_on: + x = x[:, 1:] + + B, _, C = x.shape + x = x.transpose(1, 2).reshape(B, C, thw[0], thw[1], thw[2]) + + x = self.head([x], bboxes) else: - x = x.mean(1) + if self.use_mean_pooling: + if self.cls_embed_on: + x = x[:, 1:] + x = x.mean(1) + x = self.norm(x) + elif self.cls_embed_on: + x = self.norm(x) + x = x[:, 0] + else: # this is default, [norm->mean] + x = self.norm(x) + x = x.mean(1) + x = self.head(x) - x = self.head(x) return x diff --git a/slowfast/utils/benchmark.py b/slowfast/utils/benchmark.py old mode 100755 new mode 100644 diff --git a/slowfast/utils/checkpoint.py b/slowfast/utils/checkpoint.py index 33176d4b5..46b40425e 100644 --- a/slowfast/utils/checkpoint.py +++ b/slowfast/utils/checkpoint.py @@ -4,6 +4,7 @@ """Functions that handle saving and loading of checkpoints.""" import copy +import math import numpy as np import os import pickle @@ -197,6 +198,7 @@ def load_checkpoint( convert_from_caffe2=False, epoch_reset=False, clear_name_pattern=(), + image_init=False, ): """ Load the checkpoint from the given file. If inflation is True, inflate the @@ -327,12 +329,206 @@ def load_checkpoint( pre_train_dict = checkpoint["model_state"] model_dict = ms.state_dict() + + if image_init: + if ( + "pos_embed" in pre_train_dict.keys() + and "pos_embed_xy" in model_dict.keys() + ): + print( + pre_train_dict["pos_embed"].shape, + model_dict["pos_embed_xy"].shape, + model_dict["pos_embed_class"].shape, + ) + if ( + pre_train_dict["pos_embed"].shape[1] + == model_dict["pos_embed_xy"].shape[1] + 1 + ): + pre_train_dict["pos_embed_xy"] = pre_train_dict[ + "pos_embed" + ][:, 1:] + pre_train_dict["pos_embed_class"] = pre_train_dict[ + "pos_embed" + ][:, :1] + + if ( + "patch_embed.proj.weight" in pre_train_dict.keys() + and "patch_embed.proj.weight" in model_dict.keys() + ): + print( + pre_train_dict["patch_embed.proj.weight"].shape, + model_dict["patch_embed.proj.weight"].shape, + ) + if ( + len(pre_train_dict["patch_embed.proj.weight"].shape) + == 4 + and len(model_dict["patch_embed.proj.weight"].shape) + == 5 + ): # img->video + t = model_dict["patch_embed.proj.weight"].shape[2] + pre_train_dict[ + "patch_embed.proj.weight" + ] = pre_train_dict["patch_embed.proj.weight"][ + :, :, None, :, : + ].repeat( + 1, 1, t, 1, 1 + ) + logger.info( + f"inflate patch_embed.proj.weight to {pre_train_dict['patch_embed.proj.weight'].shape}" + ) + elif ( + len(pre_train_dict["patch_embed.proj.weight"].shape) + == 5 + and len(model_dict["patch_embed.proj.weight"].shape) + == 4 + ): # video->img + orig_shape = pre_train_dict[ + "patch_embed.proj.weight" + ].shape + # pre_train_dict["patch_embed.proj.weight"] = pre_train_dict["patch_embed.proj.weight"][:, :, orig_shape[2]//2, :, :] # take center + pre_train_dict[ + "patch_embed.proj.weight" + ] = pre_train_dict["patch_embed.proj.weight"].sum( + 2 + ) # take avg + logger.info( + f"deflate patch_embed.proj.weight from {orig_shape} to {pre_train_dict['patch_embed.proj.weight'].shape}" + ) + if ( + "pos_embed_spatial" in pre_train_dict.keys() + and "pos_embed" in model_dict.keys() + ): + pos_embds = pre_train_dict["pos_embed_spatial"] + if ( + "pos_embed_class" in pre_train_dict.keys() + and pos_embds.shape + != model_dict["pos_embed"].shape + ): + pos_embds = torch.cat( + [ + pre_train_dict["pos_embed_class"], + pos_embds, + ], + 1, + ) + pre_train_dict.pop("pos_embed_class") + if pos_embds.shape == model_dict["pos_embed"].shape: + pre_train_dict["pos_embed"] = pos_embds + pre_train_dict.pop("pos_embed_spatial") + logger.info( + f"successful surgery of pos embed w/ shape {pos_embds.shape} " + ) + else: + logger.info( + f"UNSUCCESSFUL surgery of pos embed w/ shape {pos_embds.shape} " + ) + + qkv = [ + "attn.pool_k.weight", + "attn.pool_q.weight", + "attn.pool_v.weight", + ] + for k in pre_train_dict.keys(): + if ( + any([x in k for x in qkv]) + and pre_train_dict[k].shape != model_dict[k].shape + ): + # print(pre_train_dict[k].shape, model_dict[k].shape) + logger.info( + f"inflate {k} from {pre_train_dict[k].shape} to {model_dict[k].shape}" + ) + t = model_dict[k].shape[2] + pre_train_dict[k] = pre_train_dict[k].repeat( + 1, 1, t, 1, 1 + ) + + for k in pre_train_dict.keys(): + if ( + "rel_pos" in k + and pre_train_dict[k].shape != model_dict[k].shape + ): + # print(pre_train_dict[k].shape, model_dict[k].shape) + logger.info( + f"interpolating {k} from {pre_train_dict[k].shape} to {model_dict[k].shape}" + ) + new_pos_embed = torch.nn.functional.interpolate( + pre_train_dict[k] + .reshape(1, pre_train_dict[k].shape[0], -1) + .permute(0, 2, 1), + size=model_dict[k].shape[0], + mode="linear", + ) + new_pos_embed = ( + new_pos_embed.reshape(-1, model_dict[k].shape[0]) + .permute(1, 0) + .squeeze() + ) + pre_train_dict[k] = new_pos_embed + # Match pre-trained weights that have same shape as current model. - pre_train_dict_match = { - k: v - for k, v in pre_train_dict.items() - if k in model_dict and v.size() == model_dict[k].size() - } + pre_train_dict_match = {} + not_used_layers = [] + for k, v in pre_train_dict.items(): + if k in model_dict: + if v.size() == model_dict[k].size(): + pre_train_dict_match[k] = v + else: + if "attn.rel_pos" in k: + v_shape = v.shape + v = v.t().unsqueeze(0) + v = torch.nn.functional.interpolate( + v, + size=model_dict[k].size()[0], + mode="linear", + ) + v = v[0].t() + pre_train_dict_match[k] = v + logger.info( + "{} reshaped from {} to {}".format( + k, v_shape, v.shape + ) + ) + elif "pos_embed_temporal" in k: + v_shape = v.shape + v = torch.nn.functional.interpolate( + v.permute(0, 2, 1), + size=model_dict[k].shape[1], + mode="linear", + ) + pre_train_dict_match[k] = v.permute(0, 2, 1) + logger.info( + "{} reshaped from {} to {}".format( + k, v_shape, pre_train_dict_match[k].shape + ) + ) + elif "pos_embed_spatial" in k: + v_shape = v.shape + pretrain_size = int(math.sqrt(v_shape[1])) + model_size = int(math.sqrt(model_dict[k].shape[1])) + assert pretrain_size * pretrain_size == v_shape[1] + assert ( + model_size * model_size + == model_dict[k].shape[1] + ) + v = torch.nn.functional.interpolate( + v.reshape( + 1, pretrain_size, pretrain_size, -1 + ).permute(0, 3, 1, 2), + size=(model_size, model_size), + mode="bicubic", + ) + pre_train_dict_match[k] = v.reshape( + 1, -1, model_size * model_size + ).permute(0, 2, 1) + logger.info( + "{} reshaped from {} to {}".format( + k, v_shape, pre_train_dict_match[k].shape + ) + ) + else: + not_used_layers.append(k) + else: + not_used_layers.append(k) # Weights that do not have match from the pre-trained model. not_load_layers = [ k @@ -343,8 +539,15 @@ def load_checkpoint( if not_load_layers: for k in not_load_layers: logger.info("Network weights {} not loaded.".format(k)) + if not_used_layers: + for k in not_used_layers: + logger.info("Network weights {} not used.".format(k)) # Load pre-trained weights. - ms.load_state_dict(pre_train_dict_match, strict=False) + missing_keys, unexpected_keys = ms.load_state_dict( + pre_train_dict_match, strict=False + ) + print("missing keys: {}".format(missing_keys)) + print("unexpected keys: {}".format(unexpected_keys)) epoch = -1 # Load the optimizer state (commonly not done when fine-tuning) @@ -527,6 +730,7 @@ def load_train_checkpoint(cfg, model, optimizer, scaler=None): convert_from_caffe2=cfg.TRAIN.CHECKPOINT_TYPE == "caffe2", epoch_reset=cfg.TRAIN.CHECKPOINT_EPOCH_RESET, clear_name_pattern=cfg.TRAIN.CHECKPOINT_CLEAR_NAME_PATTERN, + image_init=cfg.TRAIN.CHECKPOINT_IN_INIT, ) start_epoch = checkpoint_epoch + 1 else: diff --git a/slowfast/utils/logging.py b/slowfast/utils/logging.py index a19db4960..0da650a1f 100644 --- a/slowfast/utils/logging.py +++ b/slowfast/utils/logging.py @@ -91,13 +91,13 @@ def log_json_stats(stats, output_dir=None): stats (dict): a dictionary of statistical information to log. """ stats = { - k: decimal.Decimal("{:.3f}".format(v)) if isinstance(v, float) else v + k: decimal.Decimal("{:.5f}".format(v)) if isinstance(v, float) else v for k, v in stats.items() } json_stats = simplejson.dumps(stats, sort_keys=True, use_decimal=True) logger = get_logger(__name__) logger.info("json_stats: {:s}".format(json_stats)) - if du.is_root_proc() and output_dir: + if du.is_master_proc(du.get_world_size()) and output_dir: filename = os.path.join(output_dir, "json_stats.log") try: with pathmgr.open( diff --git a/slowfast/utils/meters.py b/slowfast/utils/meters.py index 48ff1ef47..5b94f8d55 100644 --- a/slowfast/utils/meters.py +++ b/slowfast/utils/meters.py @@ -459,6 +459,9 @@ def get_win_median(self): """ return np.median(self.deque) + def get_current_value(self): + return self.deque[-1] + def get_win_avg(self): """ Calculate the current average value of the deque. @@ -472,6 +475,42 @@ def get_global_avg(self): return self.total / self.count +class ListMeter(object): + def __init__(self, list_size): + """ + Args: + list_size (int): size of the list. + """ + self.list = np.zeros(list_size) + self.total = np.zeros(list_size) + self.count = 0 + + def reset(self): + """ + Reset the meter. + """ + self.list = np.zeros_like(self.list) + self.total = np.zeros_like(self.total) + self.count = 0 + + def add_value(self, value): + """ + Add a new list value to the meter. + """ + self.list = np.array(value) + self.count += 1 + self.total += self.list + + def get_value(self): + return self.list + + def get_global_avg(self): + """ + Calculate the global mean value. + """ + return self.total / self.count + + class TrainMeter(object): """ Measure training stats. @@ -492,6 +531,7 @@ def __init__(self, epoch_iters, cfg): self.loss = ScalarMeter(cfg.LOG_PERIOD) self.loss_total = 0.0 self.lr = None + self.grad_norm = None # Current minibatch errors (smoothed over a window). self.mb_top1_err = ScalarMeter(cfg.LOG_PERIOD) self.mb_top5_err = ScalarMeter(cfg.LOG_PERIOD) @@ -500,6 +540,7 @@ def __init__(self, epoch_iters, cfg): self.num_top5_mis = 0 self.num_samples = 0 self.output_dir = cfg.OUTPUT_DIR + self.multi_loss = None def reset(self): """ @@ -508,11 +549,14 @@ def reset(self): self.loss.reset() self.loss_total = 0.0 self.lr = None + self.grad_norm = None self.mb_top1_err.reset() self.mb_top5_err.reset() self.num_top1_mis = 0 self.num_top5_mis = 0 self.num_samples = 0 + if self.multi_loss is not None: + self.multi_loss.reset() def iter_tic(self): """ @@ -532,7 +576,9 @@ def data_toc(self): self.data_timer.pause() self.net_timer.reset() - def update_stats(self, top1_err, top5_err, loss, lr, mb_size): + def update_stats( + self, top1_err, top5_err, loss, lr, grad_norm, mb_size, multi_loss=None + ): """ Update the current stats. Args: @@ -541,9 +587,11 @@ def update_stats(self, top1_err, top5_err, loss, lr, mb_size): loss (float): loss value. lr (float): learning rate. mb_size (int): mini batch size. + multi_loss (list): a list of values for multi-tasking losses. """ self.loss.add_value(loss) self.lr = lr + self.grad_norm = grad_norm self.loss_total += loss * mb_size self.num_samples += mb_size @@ -554,6 +602,26 @@ def update_stats(self, top1_err, top5_err, loss, lr, mb_size): # Aggregate stats self.num_top1_mis += top1_err * mb_size self.num_top5_mis += top5_err * mb_size + if multi_loss: + if self.multi_loss is None: + self.multi_loss = ListMeter(len(multi_loss)) + self.multi_loss.add_value(multi_loss) + if ( + self._cfg.TRAIN.KILL_LOSS_EXPLOSION_FACTOR > 0.0 + and len(self.loss.deque) > 6 + ): + prev_loss = 0.0 + for i in range(2, 7): + prev_loss += self.loss.deque[len(self.loss.deque) - i] + if ( + loss + > self._cfg.TRAIN.KILL_LOSS_EXPLOSION_FACTOR * prev_loss / 5.0 + ): + raise RuntimeError( + "ERROR: Got Loss explosion of {} {}".format( + loss, datetime.datetime.now() + ) + ) def log_iter_stats(self, cur_epoch, cur_iter): """ @@ -580,11 +648,16 @@ def log_iter_stats(self, cur_epoch, cur_iter): "eta": eta, "loss": self.loss.get_win_median(), "lr": self.lr, + "grad_norm": self.grad_norm, "gpu_mem": "{:.2f}G".format(misc.gpu_mem_usage()), } if not self._cfg.DATA.MULTI_LABEL: stats["top1_err"] = self.mb_top1_err.get_win_median() stats["top5_err"] = self.mb_top5_err.get_win_median() + if self.multi_loss is not None: + loss_list = self.multi_loss.get_value() + for idx, loss in enumerate(loss_list): + stats["loss_" + str(idx)] = loss logging.log_json_stats(stats) def log_epoch_stats(self, cur_epoch): @@ -607,6 +680,7 @@ def log_epoch_stats(self, cur_epoch): "dt_net": self.net_timer.seconds(), "eta": eta, "lr": self.lr, + "grad_norm": self.grad_norm, "gpu_mem": "{:.2f}G".format(misc.gpu_mem_usage()), "RAM": "{:.2f}/{:.2f}G".format(*misc.cpu_mem_usage()), } @@ -617,6 +691,10 @@ def log_epoch_stats(self, cur_epoch): stats["top1_err"] = top1_err stats["top5_err"] = top5_err stats["loss"] = avg_loss + if self.multi_loss is not None: + avg_loss_list = self.multi_loss.get_global_avg() + for idx, loss in enumerate(avg_loss_list): + stats["loss_" + str(idx)] = loss logging.log_json_stats(stats, self.output_dir) diff --git a/slowfast/utils/misc.py b/slowfast/utils/misc.py index 5920546d4..48e1ffac8 100644 --- a/slowfast/utils/misc.py +++ b/slowfast/utils/misc.py @@ -9,10 +9,12 @@ from datetime import datetime import psutil import torch +import torchvision.io as io from fvcore.nn.activation_count import activation_count from fvcore.nn.flop_count import flop_count from matplotlib import pyplot as plt from torch import nn +from torchvision.utils import make_grid import slowfast.utils.logging as logging import slowfast.utils.multiprocessing as mpu @@ -90,7 +92,7 @@ def _get_model_analysis_input(cfg, use_train_input): """ rgb_dimension = 3 if use_train_input: - if cfg.TRAIN.DATASET in ["imagenet", "imagenetprefetch"]: + if "imagenet" in cfg.TRAIN.DATASET: input_tensors = torch.rand( rgb_dimension, cfg.DATA.TRAIN_CROP_SIZE, @@ -104,7 +106,7 @@ def _get_model_analysis_input(cfg, use_train_input): cfg.DATA.TRAIN_CROP_SIZE, ) else: - if cfg.TEST.DATASET in ["imagenet", "imagenetprefetch"]: + if "imagenet" in cfg.TEST.DATASET: input_tensors = torch.rand( rgb_dimension, cfg.DATA.TEST_CROP_SIZE, @@ -181,13 +183,11 @@ def log_model_info(model, cfg, use_train_input=True): log info for testing. """ logger.info("Model:\n{}".format(model)) - logger.info("Params: {:,}".format(params_count(model))) + params = params_count(model) + logger.info("Params: {:,}".format(params)) logger.info("Mem: {:,} MB".format(gpu_mem_usage())) - logger.info( - "Flops: {:,} G".format( - get_model_stats(model, cfg, "flop", use_train_input) - ) - ) + flops = get_model_stats(model, cfg, "flop", use_train_input) + logger.info("Flops: {:,} G".format(flops)) logger.info( "Activations: {:,} M".format( get_model_stats(model, cfg, "activation", use_train_input) @@ -195,6 +195,7 @@ def log_model_info(model, cfg, use_train_input=True): ) logger.info("nvidia-smi") os.system("nvidia-smi") + return flops, params def is_eval_epoch(cfg, cur_epoch, multigrid_schedule): @@ -251,6 +252,124 @@ def plot_input(tensor, bboxes=(), texts=(), path="./tmp_vis.png"): f.savefig(path) +def plot_input_normed( + tensor, + bboxes=(), + texts=(), + path="./tmp_vis.png", + folder_path="", + make_grids=False, + output_video=False, +): + """ + Plot the input tensor with the optional bounding box and save it to disk. + Args: + tensor (tensor): a tensor with shape of `NxCxHxW`. + bboxes (tuple): bounding boxes with format of [[x, y, h, w]]. + texts (tuple): a tuple of string to plot. + path (str): path to the image to save to. + """ + tensor = tensor.float() + try: + os.mkdir(folder_path) + except Exception as e: + pass + tensor = convert_normalized_images(tensor) + if output_video: + # assert make_grids, "video needs to have make_grids on" + assert tensor.ndim == 5 + sz = tensor.shape + + if make_grids: + vid = tensor.reshape([sz[0], sz[1] * sz[2], sz[3], sz[4]]) + vid = make_grid(vid, padding=8, pad_value=1.0, nrow=sz[0]) + vid = vid.reshape([sz[1], sz[2], vid.shape[1], vid.shape[2]]) + else: + vid = tensor.reshape([sz[0] * sz[1], sz[2], sz[3], sz[4]]) + + vid = vid.permute([0, 2, 3, 1]) + vid *= 255.0 + vid = vid.to(torch.uint8) + fps = 30.0 * vid.shape[0] / 64.0 + io.video.write_video(path, vid, fps, video_codec="libx264") + elif make_grids: + if tensor.ndim > 4 and tensor.shape[0] == 1: + tensor = tensor.squeeze() + nrow = 1 + elif tensor.ndim == 5: + nrow = tensor.shape[1] + tensor = tensor.reshape( + shape=(-1, tensor.shape[2], tensor.shape[3], tensor.shape[4]) + ) + vis2 = ( + make_grid(tensor, padding=8, pad_value=1.0, nrow=nrow) + .permute(1, 2, 0) + .cpu() + .numpy() + ) + plt.imsave(fname=path, arr=vis2, format="png") + else: + f, ax = plt.subplots( + nrows=tensor.shape[0], + ncols=tensor.shape[1], + figsize=(10 * tensor.shape[1], 10 * tensor.shape[0]), + ) + + if tensor.shape[0] == 1: + for i in range(tensor.shape[1]): + ax[i].axis("off") + ax[i].imshow(tensor[0][i].permute(1, 2, 0)) + # ax[1][0].axis('off') + if bboxes is not None and len(bboxes) > i: + for box in bboxes[i]: + x1, y1, x2, y2 = box + ax[i].vlines(x1, y1, y2, colors="g", linestyles="solid") + ax[i].vlines(x2, y1, y2, colors="g", linestyles="solid") + ax[i].hlines(y1, x1, x2, colors="g", linestyles="solid") + ax[i].hlines(y2, x1, x2, colors="g", linestyles="solid") + + if texts is not None and len(texts) > i: + ax[i].text(0, 0, texts[i]) + else: + for i in range(tensor.shape[0]): + for j in range(tensor.shape[1]): + ax[i][j].axis("off") + ax[i][j].imshow(tensor[i][j].permute(1, 2, 0)) + # ax[1][0].axis('off') + if bboxes is not None and len(bboxes) > i: + for box in bboxes[i]: + x1, y1, x2, y2 = box + ax[i].vlines( + x1, y1, y2, colors="g", linestyles="solid" + ) + ax[i].vlines( + x2, y1, y2, colors="g", linestyles="solid" + ) + ax[i].hlines( + y1, x1, x2, colors="g", linestyles="solid" + ) + ax[i].hlines( + y2, x1, x2, colors="g", linestyles="solid" + ) + + if texts is not None and len(texts) > i: + ax[i].text(0, 0, texts[i]) + print(f"{path}") + f.tight_layout(pad=0.0) + with pathmgr.open(path, "wb") as h: + f.savefig(h) + + +def convert_normalized_images(tensor): + + tensor = tensor * 0.225 + tensor = tensor + 0.45 + + tensor = tensor.clamp(min=0.0, max=1.0) + + return tensor + + def frozen_bn_stats(model): """ Set all the bn layers to eval mode. diff --git a/tools/benchmark.py b/tools/benchmark.py old mode 100755 new mode 100644 diff --git a/tools/run_net.py b/tools/run_net.py index 847448f69..d654b12f9 100644 --- a/tools/run_net.py +++ b/tools/run_net.py @@ -20,6 +20,7 @@ def main(): print("config files: {}".format(args.cfg_files)) for path_to_config in args.cfg_files: cfg = load_config(args, path_to_config) + cfg = assert_and_infer_cfg(cfg) # Perform training. if cfg.TRAIN.ENABLE: @@ -27,7 +28,13 @@ def main(): # Perform multi-clip testing. if cfg.TEST.ENABLE: - launch_job(cfg=cfg, init_method=args.init_method, func=test) + if cfg.TEST.NUM_ENSEMBLE_VIEWS == -1: + num_view_list = [1, 3, 5, 7, 10] + for num_view in num_view_list: + cfg.TEST.NUM_ENSEMBLE_VIEWS = num_view + launch_job(cfg=cfg, init_method=args.init_method, func=test) + else: + launch_job(cfg=cfg, init_method=args.init_method, func=test) # Perform model visualization. if cfg.TENSORBOARD.ENABLE and ( diff --git a/tools/test_net.py b/tools/test_net.py index ddf53dc02..9d32bcef9 100644 --- a/tools/test_net.py +++ b/tools/test_net.py @@ -118,7 +118,6 @@ def perform_test(test_loader, model, test_meter, cfg, writer=None): else: # Perform the forward pass. preds = model(inputs) - # Gather all the predictions across all the devices to perform ensemble. if cfg.NUM_GPUS > 1: preds, labels, video_idx = du.all_gather([preds, labels, video_idx]) @@ -128,10 +127,12 @@ def perform_test(test_loader, model, test_meter, cfg, writer=None): video_idx = video_idx.cpu() test_meter.iter_toc() - # Update and log stats. - test_meter.update_stats( - preds.detach(), labels.detach(), video_idx.detach() - ) + + if not cfg.VIS_MASK.ENABLE: + # Update and log stats. + test_meter.update_stats( + preds.detach(), labels.detach(), video_idx.detach() + ) test_meter.log_iter_stats(cur_iter) test_meter.iter_tic() @@ -177,83 +178,107 @@ def test(cfg): # Setup logging format. logging.setup_logging(cfg.OUTPUT_DIR) - # Print config. - logger.info("Test with config:") - logger.info(cfg) + if len(cfg.TEST.NUM_TEMPORAL_CLIPS) == 0: + cfg.TEST.NUM_TEMPORAL_CLIPS = [cfg.TEST.NUM_ENSEMBLE_VIEWS] - # Build the video model and print model statistics. - model = build_model(cfg) + test_meters = [] + for num_view in cfg.TEST.NUM_TEMPORAL_CLIPS: - out_str_prefix = "lin" if cfg.MODEL.DETACH_FINAL_FC else "" + cfg.TEST.NUM_ENSEMBLE_VIEWS = num_view - if du.is_master_proc() and cfg.LOG_MODEL_INFO: - misc.log_model_info(model, cfg, use_train_input=False) + # Print config. + logger.info("Test with config:") + logger.info(cfg) - if ( - cfg.TASK == "ssl" - and cfg.MODEL.MODEL_NAME == "ContrastiveModel" - and cfg.CONTRASTIVE.KNN_ON - ): - train_loader = loader.construct_loader(cfg, "train") - out_str_prefix = "knn" - if hasattr(model, "module"): - model.module.init_knn_labels(train_loader) + # Build the video model and print model statistics. + model = build_model(cfg) + flops, params = 0.0, 0.0 + if du.is_master_proc() and cfg.LOG_MODEL_INFO: + model.eval() + flops, params = misc.log_model_info( + model, cfg, use_train_input=False + ) + + if du.is_master_proc() and cfg.LOG_MODEL_INFO: + misc.log_model_info(model, cfg, use_train_input=False) + if ( + cfg.TASK == "ssl" + and cfg.MODEL.MODEL_NAME == "ContrastiveModel" + and cfg.CONTRASTIVE.KNN_ON + ): + train_loader = loader.construct_loader(cfg, "train") + if hasattr(model, "module"): + model.module.init_knn_labels(train_loader) + else: + model.init_knn_labels(train_loader) + + cu.load_test_checkpoint(cfg, model) + + # Create video testing loaders. + test_loader = loader.construct_loader(cfg, "test") + logger.info("Testing model for {} iterations".format(len(test_loader))) + + if cfg.DETECTION.ENABLE: + assert cfg.NUM_GPUS == cfg.TEST.BATCH_SIZE or cfg.NUM_GPUS == 0 + test_meter = AVAMeter(len(test_loader), cfg, mode="test") + else: + assert ( + test_loader.dataset.num_videos + % (cfg.TEST.NUM_ENSEMBLE_VIEWS * cfg.TEST.NUM_SPATIAL_CROPS) + == 0 + ) + # Create meters for multi-view testing. + test_meter = TestMeter( + test_loader.dataset.num_videos + // (cfg.TEST.NUM_ENSEMBLE_VIEWS * cfg.TEST.NUM_SPATIAL_CROPS), + cfg.TEST.NUM_ENSEMBLE_VIEWS * cfg.TEST.NUM_SPATIAL_CROPS, + cfg.MODEL.NUM_CLASSES + if not cfg.TASK == "ssl" + else cfg.CONTRASTIVE.NUM_CLASSES_DOWNSTREAM, + len(test_loader), + cfg.DATA.MULTI_LABEL, + cfg.DATA.ENSEMBLE_METHOD, + ) + + # Set up writer for logging to Tensorboard format. + if cfg.TENSORBOARD.ENABLE and du.is_master_proc( + cfg.NUM_GPUS * cfg.NUM_SHARDS + ): + writer = tb.TensorboardWriter(cfg) else: - model.init_knn_labels(train_loader) - - cu.load_test_checkpoint(cfg, model) - - # Create video testing loaders. - test_loader = loader.construct_loader(cfg, "test") - logger.info("Testing model for {} iterations".format(len(test_loader))) - - if cfg.DETECTION.ENABLE: - assert cfg.NUM_GPUS == cfg.TEST.BATCH_SIZE or cfg.NUM_GPUS == 0 - test_meter = AVAMeter(len(test_loader), cfg, mode="test") - else: - assert ( - test_loader.dataset.num_videos - % (cfg.TEST.NUM_ENSEMBLE_VIEWS * cfg.TEST.NUM_SPATIAL_CROPS) - == 0 + writer = None + + # # Perform multi-view test on the entire dataset. + test_meter = perform_test(test_loader, model, test_meter, cfg, writer) + test_meters.append(test_meter) + if writer is not None: + writer.close() + + result_string_views = "_p{:.2f}_f{:.2f}".format(params / 1e6, flops) + + for view, test_meter in zip(cfg.TEST.NUM_TEMPORAL_CLIPS, test_meters): + logger.info( + "Finalized testing with {} temporal clips and {} spatial crops".format( + view, cfg.TEST.NUM_SPATIAL_CROPS + ) ) - # Create meters for multi-view testing. - test_meter = TestMeter( - test_loader.dataset.num_videos - // (cfg.TEST.NUM_ENSEMBLE_VIEWS * cfg.TEST.NUM_SPATIAL_CROPS), - cfg.TEST.NUM_ENSEMBLE_VIEWS * cfg.TEST.NUM_SPATIAL_CROPS, - cfg.MODEL.NUM_CLASSES - if not cfg.TASK == "ssl" - else cfg.CONTRASTIVE.NUM_CLASSES_DOWNSTREAM, - len(test_loader), - cfg.DATA.MULTI_LABEL, - cfg.DATA.ENSEMBLE_METHOD, + result_string_views += "_{}a{}" "".format( + view, test_meter.stats["top1_acc"] ) - # Set up writer for logging to Tensorboard format. - if cfg.TENSORBOARD.ENABLE and du.is_master_proc( - cfg.NUM_GPUS * cfg.NUM_SHARDS - ): - writer = tb.TensorboardWriter(cfg) - else: - writer = None - - # # Perform multi-view test on the entire dataset. - test_meter = perform_test(test_loader, model, test_meter, cfg, writer) - if writer is not None: - writer.close() - result_string = ( - "_a{}{}{} Top1 Acc: {} Top5 Acc: {} MEM: {:.2f} dataset: {}{}" - "".format( - out_str_prefix, - cfg.TEST.DATASET[0], - test_meter.stats["top1_acc"], - test_meter.stats["top1_acc"], - test_meter.stats["top5_acc"], - misc.gpu_mem_usage(), - cfg.TEST.DATASET[0], - cfg.MODEL.NUM_CLASSES, + result_string = ( + "_p{:.2f}_f{:.2f}_{}a{} Top5 Acc: {} MEM: {:.2f} f: {:.4f}" + "".format( + params / 1e6, + flops, + view, + test_meter.stats["top1_acc"], + test_meter.stats["top5_acc"], + misc.gpu_mem_usage(), + flops, + ) ) - ) - logger.info("testing done: {}".format(result_string)) - return result_string + logger.info("{}".format(result_string)) + logger.info("{}".format(result_string_views)) + return result_string + " \n " + result_string_views diff --git a/tools/train_net.py b/tools/train_net.py index cfc75aff4..b4707a4d1 100644 --- a/tools/train_net.py +++ b/tools/train_net.py @@ -20,7 +20,10 @@ from slowfast.datasets import loader from slowfast.datasets.mixup import MixUp from slowfast.models import build_model -from slowfast.models.contrastive import cancel_swav_gradients +from slowfast.models.contrastive import ( + contrastive_forward, + contrastive_parameter_surgery, +) from slowfast.utils.meters import AVAMeter, EpochTimer, TrainMeter, ValMeter from slowfast.utils.multigrid import MultigridSchedule @@ -66,18 +69,6 @@ def train_epoch( num_classes=cfg.MODEL.NUM_CLASSES, ) - iters_noupdate = 0 - if ( - cfg.MODEL.MODEL_NAME == "ContrastiveModel" - and cfg.CONTRASTIVE.TYPE == "moco" - ): - assert ( - cfg.CONTRASTIVE.QUEUE_LEN % (cfg.TRAIN.BATCH_SIZE * cfg.NUM_SHARDS) - == 0 - ) - iters_noupdate = ( - cfg.CONTRASTIVE.QUEUE_LEN // cfg.TRAIN.BATCH_SIZE // cfg.NUM_SHARDS - ) if cfg.MODEL.FROZEN_BN: misc.frozen_bn_stats(model) # Explicitly declare reduction to mean. @@ -97,15 +88,17 @@ def train_epoch( inputs[i] = inputs[i].cuda(non_blocking=True) else: inputs = inputs.cuda(non_blocking=True) - labels = labels.cuda() + if not isinstance(labels, list): + labels = labels.cuda(non_blocking=True) + index = index.cuda(non_blocking=True) + time = time.cuda(non_blocking=True) for key, val in meta.items(): if isinstance(val, (list,)): for i in range(len(val)): val[i] = val[i].cuda(non_blocking=True) else: meta[key] = val.cuda(non_blocking=True) - index = index.cuda() - time = time.cuda() + batch_size = ( inputs[0][0].size(0) if isinstance(inputs[0], list) @@ -139,43 +132,47 @@ def train_epoch( elif cfg.DETECTION.ENABLE: # Compute the predictions. preds = model(inputs, meta["boxes"]) + elif cfg.MASK.ENABLE: + preds, labels = model(inputs) else: preds = model(inputs) - if cfg.TASK == "ssl" and cfg.MODEL.MODEL_NAME == "ContrastiveModel": - labels = torch.zeros( - preds.size(0), dtype=labels.dtype, device=labels.device - ) + if cfg.TASK == "ssl" and cfg.MODEL.MODEL_NAME == "ContrastiveModel": + labels = torch.zeros( + preds.size(0), dtype=labels.dtype, device=labels.device + ) - if cfg.MODEL.MODEL_NAME == "ContrastiveModel" and partial_loss: - loss = partial_loss - else: - # Compute the loss. - loss = loss_fun(preds, labels) + if cfg.MODEL.MODEL_NAME == "ContrastiveModel" and partial_loss: + loss = partial_loss + else: + # Compute the loss. + loss = loss_fun(preds, labels) + + loss_extra = None + if isinstance(loss, (list, tuple)): + loss, loss_extra = loss # check Nan Loss. misc.check_nan_losses(loss) - if perform_backward: scaler.scale(loss).backward() # Unscales the gradients of optimizer's assigned params in-place scaler.unscale_(optimizer) # Clip gradients if necessary if cfg.SOLVER.CLIP_GRAD_VAL: - torch.nn.utils.clip_grad_value_( + grad_norm = torch.nn.utils.clip_grad_value_( model.parameters(), cfg.SOLVER.CLIP_GRAD_VAL ) elif cfg.SOLVER.CLIP_GRAD_L2NORM: - torch.nn.utils.clip_grad_norm_( + grad_norm = torch.nn.utils.clip_grad_norm_( model.parameters(), cfg.SOLVER.CLIP_GRAD_L2NORM ) - - model = cancel_swav_gradients(model, cfg, epoch_exact) - if cur_iter < iters_noupdate and cur_epoch == 0: # for e.g. MoCo - logger.info( - "Not updating parameters {}/{}".format(cur_iter, iters_noupdate) - ) else: - # Update the parameters. + grad_norm = optim.get_grad_norm_(model.parameters()) + # Update the parameters. (defaults to True) + model, update_param = contrastive_parameter_surgery( + model, cfg, epoch_exact, cur_iter + ) + if update_param: scaler.step(optimizer) scaler.update() @@ -209,8 +206,25 @@ def train_epoch( if cfg.DATA.MULTI_LABEL: # Gather all the predictions across all the devices. if cfg.NUM_GPUS > 1: - [loss] = du.all_reduce([loss]) - loss = loss.item() + loss, grad_norm = du.all_reduce([loss, grad_norm]) + loss, grad_norm = ( + loss.item(), + grad_norm.item(), + ) + elif cfg.MASK.ENABLE: + # Gather all the predictions across all the devices. + if cfg.NUM_GPUS > 1: + loss, grad_norm = du.all_reduce([loss, grad_norm]) + if loss_extra: + loss_extra = du.all_reduce(loss_extra) + loss, grad_norm, top1_err, top5_err = ( + loss.item(), + grad_norm.item(), + 0.0, + 0.0, + ) + if loss_extra: + loss_extra = [one_loss.item() for one_loss in loss_extra] else: # Compute the errors. num_topks_correct = metrics.topks_correct(preds, labels, (1, 5)) @@ -219,13 +233,14 @@ def train_epoch( ] # Gather all the predictions across all the devices. if cfg.NUM_GPUS > 1: - loss, top1_err, top5_err = du.all_reduce( - [loss.detach(), top1_err, top5_err] + loss, grad_norm, top1_err, top5_err = du.all_reduce( + [loss.detach(), grad_norm, top1_err, top5_err] ) # Copy the stats from GPU to CPU (sync point). - loss, top1_err, top5_err = ( + loss, grad_norm, top1_err, top5_err = ( loss.item(), + grad_norm.item(), top1_err.item(), top5_err.item(), ) @@ -236,10 +251,12 @@ def train_epoch( top5_err, loss, lr, + grad_norm, batch_size * max( cfg.NUM_GPUS, 1 ), # If running on CPU (cfg.NUM_GPUS == 1), use 1 to represent 1 CPU. + loss_extra, ) # write to tensorboard format if available. if writer is not None: @@ -252,8 +269,6 @@ def train_epoch( }, global_step=data_size * cur_epoch + cur_iter, ) - - torch.cuda.synchronize() train_meter.iter_toc() # do measure allreduce for this meter train_meter.log_iter_stats(cur_epoch, cur_iter) torch.cuda.synchronize() @@ -360,6 +375,8 @@ def eval_epoch( if cfg.NUM_GPUS > 1: preds, labels = du.all_gather([preds, labels]) else: + if cfg.DATA.IN22k_VAL_IN1K != "": + preds = preds[:, :1000] # Compute the errors. num_topks_correct = metrics.topks_correct(preds, labels, (1, 5)) @@ -418,56 +435,6 @@ def eval_epoch( val_meter.reset() -def contrastive_forward(model, cfg, inputs, index, time, epoch_exact, scaler): - if cfg.CONTRASTIVE.SEQUENTIAL: - perform_backward = False - mdl = model.module if hasattr(model, "module") else model - keys = ( - mdl.compute_key_feat( - inputs, - compute_predictor_keys=False, - batched_inference=True if len(inputs) < 2 else False, - ) - if cfg.CONTRASTIVE.TYPE == "moco" or cfg.CONTRASTIVE.TYPE == "byol" - else [None] * len(inputs) - ) - for k, vid in enumerate(inputs): - other_keys = keys[:k] + keys[k + 1 :] - time_cur = torch.cat( - [ - time[:, k : k + 1, :], - time[:, :k, :], - time[:, k + 1 :, :], - ], - 1, - ) # q, kpre, kpost - vids = [vid] - if ( - cfg.CONTRASTIVE.TYPE == "swav" - or cfg.CONTRASTIVE.TYPE == "simclr" - ): - if k < len(inputs) - 1: - vids = inputs[k : k + 2] - else: - break - lgt_k, loss_k = model( - vids, index, time_cur, epoch_exact, keys=other_keys - ) - scaler.scale(loss_k).backward() - if k == 0: - preds, partial_loss = lgt_k, loss_k.detach() - else: - preds = torch.cat([preds, lgt_k], dim=0) - partial_loss += loss_k.detach() - partial_loss /= len(inputs) * 2.0 # to have same loss as symm model - if cfg.CONTRASTIVE.TYPE == "moco": - mdl._dequeue_and_enqueue(keys) - else: - perform_backward = True - preds, partial_loss = model(inputs, index, time, epoch_exact, keys=None) - return model, preds, partial_loss, perform_backward - - def calculate_and_update_precise_bn(loader, model, num_iters=200, use_gpu=True): """ Update the stats in bn layers by calculate the precise stats. @@ -512,7 +479,7 @@ def build_trainer(cfg): # Build the video model and print model statistics. model = build_model(cfg) if du.is_master_proc() and cfg.LOG_MODEL_INFO: - misc.log_model_info(model, cfg, use_train_input=True) + flops, params = misc.log_model_info(model, cfg, use_train_input=True) # Construct the optimizer. optimizer = optim.construct_optimizer(model, cfg) @@ -567,8 +534,9 @@ def train(cfg): # Build the video model and print model statistics. model = build_model(cfg) + flops, params = 0.0, 0.0 if du.is_master_proc() and cfg.LOG_MODEL_INFO: - misc.log_model_info(model, cfg, use_train_input=True) + flops, params = misc.log_model_info(model, cfg, use_train_input=True) # Construct the optimizer. optimizer = optim.construct_optimizer(model, cfg) @@ -614,6 +582,7 @@ def train(cfg): convert_from_caffe2=cfg.TRAIN.CHECKPOINT_TYPE == "caffe2", epoch_reset=cfg.TRAIN.CHECKPOINT_EPOCH_RESET, clear_name_pattern=cfg.TRAIN.CHECKPOINT_CLEAR_NAME_PATTERN, + IN_INIT=cfg.TRAIN.CHECKPOINT_IN_INIT, ) start_epoch = checkpoint_epoch + 1 else: @@ -737,8 +706,13 @@ def train(cfg): ) or cur_epoch == cfg.SOLVER.MAX_EPOCH - 1 ) - is_eval_epoch = misc.is_eval_epoch( - cfg, cur_epoch, None if multigrid is None else multigrid.schedule + is_eval_epoch = ( + misc.is_eval_epoch( + cfg, + cur_epoch, + None if multigrid is None else multigrid.schedule, + ) + and not cfg.MASK.ENABLE ) # Compute precise BN stats. @@ -778,10 +752,20 @@ def train(cfg): ) if writer is not None: writer.close() - result_string = "Top1 Acc: {:.2f} Top5 Acc: {:.2f} MEM: {:.2f}" "".format( - 100 - val_meter.min_top1_err, - 100 - val_meter.min_top5_err, - misc.gpu_mem_usage(), + result_string = ( + "_p{:.2f}_f{:.2f} _t{:.2f}_m{:.2f} _a{:.2f} Top5 Acc: {:.2f} MEM: {:.2f} f: {:.4f}" + "".format( + params / 1e6, + flops, + epoch_timer.median_epoch_time() / 60.0 + if len(epoch_timer.epoch_times) + else 0.0, + misc.gpu_mem_usage(), + 100 - val_meter.min_top1_err, + 100 - val_meter.min_top5_err, + misc.gpu_mem_usage(), + flops, + ) ) logger.info("training done: {}".format(result_string)) From 7f1327db6bd274f22fccddcfaeb32113e7dd757e Mon Sep 17 00:00:00 2001 From: Christoph Feichtenhofer Date: Fri, 5 Aug 2022 17:58:08 -0700 Subject: [PATCH 04/34] fix MaskFeat readme Reviewed By: haooooooqi Differential Revision: D38480486 fbshipit-source-id: 414825c5114b72c830aee187d427522d7edbc659 --- MODEL_ZOO.md | 2 +- projects/maskfeat/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/MODEL_ZOO.md b/MODEL_ZOO.md index 8971c6a0a..cc46af5fe 100644 --- a/MODEL_ZOO.md +++ b/MODEL_ZOO.md @@ -68,7 +68,7 @@ We also release the imagenet pretrained model if finetuning from ImageNet is pre | architecture | size | Top1 | Top5 | model | Config | | ------------- | ------------- | ------------- | ------------- | ------------- | ------------- | -| ResNet | R50 | 73.7 | 93.2 | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/model_zoo/kinetics400/R50_IN1K.pyth) | ImageNet/RES_R50 | +| ResNet | R50 | 76.4 | 93.2 | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/model_zoo/kinetics400/R50_IN1K.pyth) | ImageNet/RES_R50 | | MVIT | B-16-Conv | 82.9 | 96.3 | [`link`](https://drive.google.com/file/d/1dYYqUB-3DSgBVc9d6o-rW8ojtVsrFLgp/view?usp=sharing) | ImageNet/MVIT_B_16_CONV | diff --git a/projects/maskfeat/README.md b/projects/maskfeat/README.md index 0c1e4109a..6c6678606 100644 --- a/projects/maskfeat/README.md +++ b/projects/maskfeat/README.md @@ -15,7 +15,7 @@ In CVPR, 2022. [[Paper](https://arxiv.org/abs/2112.09133)] | name | top1 | config pre-train (PT) | config fine-tune | model PT | | ------------- | ------------- | ------------- | ------------- | ------------- | -| ViT-B | 81.3 | in1k_VIT_B_MaskFeat_PT | in1k_VIT_B_MaskFeat_FT | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/masked_models/in1k_VIT_B_MaskFeat_PT_epoch_01600.pyth) | +| ViT-B | 84.0 | in1k_VIT_B_MaskFeat_PT | in1k_VIT_B_MaskFeat_FT | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/masked_models/in1k_VIT_B_MaskFeat_PT_epoch_01600.pyth) | | ViT-L | 85.7 | in1k_VIT_L_MaskFeat_PT | in1k_VIT_L_MaskFeat_FT | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/masked_models/in1k_VIT_L_MaskFeat_PT_epoch_01600.pyth) | From 73c3ee697814bd184c9791659e9454779a552267 Mon Sep 17 00:00:00 2001 From: Yanghao Li Date: Fri, 19 Aug 2022 22:27:58 -0700 Subject: [PATCH 05/34] ssv2 Summary: releasing ssv2 dataset for mvitv2 with configs and models Reviewed By: feichtenhofer Differential Revision: D38195788 fbshipit-source-id: 2375a375237072ce51693cda16735321595043a0 --- configs/ImageNet/MVITv2_B.yaml | 2 +- configs/SSv2/MVITv2_B_32x3.yaml | 102 +++++++++++++++++++++++++++ configs/SSv2/MVITv2_L_40x3.yaml | 118 ++++++++++++++++++++++++++++++++ configs/SSv2/MVITv2_S_16x4.yaml | 99 +++++++++++++++++++++++++++ projects/mvitv2/README.md | 10 +++ slowfast/datasets/ssv2.py | 77 ++++++++++++++++----- slowfast/datasets/utils.py | 102 +++++++++++++++++++++++++++ tools/train_net.py | 2 +- 8 files changed, 494 insertions(+), 18 deletions(-) create mode 100644 configs/SSv2/MVITv2_B_32x3.yaml create mode 100644 configs/SSv2/MVITv2_L_40x3.yaml create mode 100644 configs/SSv2/MVITv2_S_16x4.yaml diff --git a/configs/ImageNet/MVITv2_B.yaml b/configs/ImageNet/MVITv2_B.yaml index c6aea74bd..5d0a330e7 100644 --- a/configs/ImageNet/MVITv2_B.yaml +++ b/configs/ImageNet/MVITv2_B.yaml @@ -25,7 +25,7 @@ MVIT: NUM_HEADS: 1 MLP_RATIO: 4.0 QKV_BIAS: True -DROPPATH_RATE: 0.3 + DROPPATH_RATE: 0.3 DEPTH: 24 NORM: "layernorm" DIM_MUL: [[2, 2.0], [5, 2.0], [21, 2.0]] diff --git a/configs/SSv2/MVITv2_B_32x3.yaml b/configs/SSv2/MVITv2_B_32x3.yaml new file mode 100644 index 000000000..ff0db83de --- /dev/null +++ b/configs/SSv2/MVITv2_B_32x3.yaml @@ -0,0 +1,102 @@ +TRAIN: + ENABLE: True + DATASET: ssv2 + BATCH_SIZE: 16 + EVAL_PERIOD: 2 + CHECKPOINT_PERIOD: 2 + AUTO_RESUME: True + # CHECKPOINT_FILE_PATH: + CHECKPOINT_TYPE: pytorch + CHECKPOINT_EPOCH_RESET: True +DATA: + DECODING_BACKEND: torchvision + NUM_FRAMES: 32 + SAMPLING_RATE: 3 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 224 + INPUT_CHANNEL_NUM: [3] + INV_UNIFORM_SAMPLE: True + RANDOM_FLIP: False + REVERSE_INPUT_CHANNEL: True + # PATH_TO_DATA_DIR: +MVIT: + ZERO_DECAY_POS_CLS: False + USE_ABS_POS: False + REL_POS_SPATIAL: True + REL_POS_TEMPORAL: True + DEPTH: 24 + NUM_HEADS: 1 + EMBED_DIM: 96 + PATCH_KERNEL: (3, 7, 7) + PATCH_STRIDE: (2, 4, 4) + PATCH_PADDING: (1, 3, 3) + MLP_RATIO: 4.0 + QKV_BIAS: True + DROPPATH_RATE: 0.4 + NORM: "layernorm" + MODE: "conv" + CLS_EMBED_ON: True + DIM_MUL: [[2, 2.0], [5, 2.0], [21, 2.0]] + HEAD_MUL: [[2, 2.0], [5, 2.0], [21, 2.0]] + POOL_KVQ_KERNEL: [3, 3, 3] + POOL_KV_STRIDE_ADAPTIVE: [1, 8, 8] + POOL_Q_STRIDE: [[0, 1, 1, 1], [1, 1, 1, 1], [2, 1, 2, 2], [3, 1, 1, 1], [4, 1, 1, 1], [5, 1, 2, 2], [6, 1, 1, 1], [7, 1, 1, 1], [8, 1, 1, 1], [9, 1, 1, 1], [10, 1, 1, 1], [11, 1, 1, 1], [12, 1, 1, 1], [13, 1, 1, 1], [14, 1, 1, 1], [15, 1, 1, 1], [16, 1, 1, 1], [17, 1, 1, 1], [18, 1, 1, 1], [19, 1, 1, 1], [20, 1, 1, 1], [21, 1, 2, 2], [22, 1, 1, 1], [23, 1, 1, 1]] + DROPOUT_RATE: 0.0 + DIM_MUL_IN_ATT: True + RESIDUAL_POOLING: True +AUG: + NUM_SAMPLE: 1 + ENABLE: True + COLOR_JITTER: 0.4 + AA_TYPE: rand-m7-n4-mstd0.5-inc1 + INTERPOLATION: bicubic + RE_PROB: 0.25 + RE_MODE: pixel + RE_COUNT: 1 + RE_SPLIT: False +MIXUP: + ENABLE: True + ALPHA: 0.8 + CUTMIX_ALPHA: 1.0 + PROB: 1.0 + SWITCH_PROB: 0.5 + LABEL_SMOOTH_VALUE: 0.1 +BN: + USE_PRECISE_STATS: False + NUM_BATCHES_PRECISE: 200 +SOLVER: + ZERO_WD_1D_PARAM: True + BASE_LR_SCALE_NUM_SHARDS: True + BASE_LR: 0.0015 + COSINE_AFTER_WARMUP: True + COSINE_END_LR: 1e-6 + WARMUP_START_LR: 1e-4 + WARMUP_EPOCHS: 3.0 + LR_POLICY: cosine + MAX_EPOCH: 100 + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-4 + OPTIMIZING_METHOD: sgd + COSINE_AFTER_WARMUP: True +MODEL: + NUM_CLASSES: 174 + ARCH: mvit + MODEL_NAME: MViT + LOSS_FUNC: soft_cross_entropy + DROPOUT_RATE: 0.5 +TEST: + ENABLE: True + DATASET: ssv2 + BATCH_SIZE: 64 + NUM_ENSEMBLE_VIEWS: 1 + NUM_SPATIAL_CROPS: 3 +DATA_LOADER: + NUM_WORKERS: 2 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . +PREFETCH: + NUM_LOADERS: 3 diff --git a/configs/SSv2/MVITv2_L_40x3.yaml b/configs/SSv2/MVITv2_L_40x3.yaml new file mode 100644 index 000000000..a3e918d19 --- /dev/null +++ b/configs/SSv2/MVITv2_L_40x3.yaml @@ -0,0 +1,118 @@ +TRAIN: + ENABLE: True + DATASET: ssv2 + BATCH_SIZE: 8 + EVAL_PERIOD: 2 + CHECKPOINT_PERIOD: 1 + AUTO_RESUME: True + #CHECKPOINT_FILE_PATH: + CHECKPOINT_IN_INIT: True + CHECKPOINT_TYPE: pytorch + CHECKPOINT_EPOCH_RESET: True +DATA: + DECODING_BACKEND: torchvision + NUM_FRAMES: 40 + SAMPLING_RATE: 3 + TRAIN_JITTER_SCALES: [356, 446] + TRAIN_CROP_SIZE: 312 + TEST_CROP_SIZE: 312 + INPUT_CHANNEL_NUM: [3] + INV_UNIFORM_SAMPLE: True + RANDOM_FLIP: False + REVERSE_INPUT_CHANNEL: True + # PATH_TO_DATA_DIR: + MEAN: [0.485, 0.456, 0.406] + STD: [0.229, 0.224, 0.225] +MVIT: + ZERO_DECAY_POS_CLS: False + USE_ABS_POS: False + REL_POS_SPATIAL: True + REL_POS_TEMPORAL: True + DEPTH: 48 + NUM_HEADS: 2 + EMBED_DIM: 144 + PATCH_KERNEL: (3, 7, 7) + PATCH_STRIDE: (2, 4, 4) + PATCH_PADDING: (1, 3, 3) + MLP_RATIO: 4.0 + QKV_BIAS: True + DROPPATH_RATE: 0.75 + NORM: "layernorm" + MODE: "conv" + CLS_EMBED_ON: True + DIM_MUL: [[2, 2.0], [8, 2.0], [44, 2.0]] + HEAD_MUL: [[2, 2.0], [8, 2.0], [44, 2.0]] + POOL_KVQ_KERNEL: [3, 3, 3] + POOL_KV_STRIDE_ADAPTIVE: [2, 4, 4] + DROPOUT_RATE: 0.0 + POOL_Q_STRIDE: [[2, 1, 2, 2], [8, 1, 2, 2], [44, 1, 2, 2]] + RESIDUAL_POOLING: True + MODE: "conv" + DIM_MUL_IN_ATT: True + POOL_Q_STRIDE: [[0, 1, 1, 1], [1, 1, 1, 1], [2, 1, 2, 2], [3, 1, 1, 1], [4, 1, 1, 1], [5, 1, 1, 1], [6, 1, 1, 1], [7, 1, 1, 1], [8, 1, 2, 2], [9, 1, 1, 1], [10, 1, 1, 1], + [11, 1, 1, 1], [12, 1, 1, 1], [13, 1, 1, 1], [14, 1, 1, 1], [15, 1, 1, 1], [16, 1, 1, 1], [17, 1, 1, 1], [18, 1, 1, 1], [19, 1, 1, 1], [20, 1, 1, 1], + [21, 1, 1, 1], [22, 1, 1, 1], [23, 1, 1, 1], [24, 1, 1, 1], [25, 1, 1, 1], [26, 1, 1, 1], [27, 1, 1, 1], [28, 1, 1, 1], [29, 1, 1, 1], [30, 1, 1, 1], + [31, 1, 1, 1], [32, 1, 1, 1], [33, 1, 1, 1], [34, 1, 1, 1], [35, 1, 1, 1], [36, 1, 1, 1], [37, 1, 1, 1], [38, 1, 1, 1], [39, 1, 1, 1], [40, 1, 1, 1], + [41, 1, 1, 1], [42, 1, 1, 1], [43, 1, 1, 1], [44, 1, 2, 2], [45, 1, 1, 1], [46, 1, 1, 1], [47, 1, 1, 1] ] + POOL_FIRST: False # default: False + POOL_KV_STRIDE_ADAPTIVE: [1, 8, 8] + +AUG: + NUM_SAMPLE: 1 + ENABLE: True + COLOR_JITTER: 0.4 + AA_TYPE: rand-m7-n4-mstd0.5-inc1 + INTERPOLATION: bicubic + RE_PROB: 0.25 + RE_MODE: pixel + RE_COUNT: 1 + RE_SPLIT: False +MIXUP: + ENABLE: True + ALPHA: 0.8 + CUTMIX_ALPHA: 1.0 + PROB: 0.0 + SWITCH_PROB: 0.5 + LABEL_SMOOTH_VALUE: 0.1 +BN: + USE_PRECISE_STATS: False + NUM_BATCHES_PRECISE: 200 +SOLVER: + CLIP_GRAD_L2NORM: 2.0 + ZERO_WD_1D_PARAM: True + BASE_LR_SCALE_NUM_SHARDS: True + BASE_LR: 0.00125 + # BASE_LR: 0.0001 + + COSINE_AFTER_WARMUP: True + COSINE_END_LR: 1e-6 + WARMUP_START_LR: 1e-6 + WARMUP_EPOCHS: 3.0 + LR_POLICY: cosine + MAX_EPOCH: 40 + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-4 + OPTIMIZING_METHOD: sgd + COSINE_AFTER_WARMUP: True +MODEL: + NUM_CLASSES: 174 + ARCH: mvit + MODEL_NAME: MViT + LOSS_FUNC: soft_cross_entropy + DROPOUT_RATE: 0.5 + ACT_CHECKPOINT: True +TEST: + ENABLE: True + DATASET: ssv2 + BATCH_SIZE: 8 + NUM_ENSEMBLE_VIEWS: 1 + NUM_SPATIAL_CROPS: 3 +DATA_LOADER: + NUM_WORKERS: 2 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . +PREFETCH: + NUM_LOADERS: 3 diff --git a/configs/SSv2/MVITv2_S_16x4.yaml b/configs/SSv2/MVITv2_S_16x4.yaml new file mode 100644 index 000000000..5fd28fe41 --- /dev/null +++ b/configs/SSv2/MVITv2_S_16x4.yaml @@ -0,0 +1,99 @@ +TRAIN: + ENABLE: True + DATASET: ssv2 + BATCH_SIZE: 16 + EVAL_PERIOD: 2 + CHECKPOINT_PERIOD: 2 + AUTO_RESUME: True + # CHECKPOINT_FILE_PATH: + CHECKPOINT_TYPE: pytorch + CHECKPOINT_EPOCH_RESET: True +DATA: + DECODING_BACKEND: torchvision + NUM_FRAMES: 16 + SAMPLING_RATE: 4 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 224 + INPUT_CHANNEL_NUM: [3] + INV_UNIFORM_SAMPLE: True + RANDOM_FLIP: False + REVERSE_INPUT_CHANNEL: True + #PATH_TO_DATA_DIR: +MVIT: + ZERO_DECAY_POS_CLS: False + USE_ABS_POS: False + REL_POS_SPATIAL: True + REL_POS_TEMPORAL: True + DEPTH: 16 + NUM_HEADS: 1 + EMBED_DIM: 96 + PATCH_KERNEL: (3, 7, 7) + PATCH_STRIDE: (2, 4, 4) + PATCH_PADDING: (1, 3, 3) + MLP_RATIO: 4.0 + QKV_BIAS: True + DROPPATH_RATE: 0.4 + NORM: "layernorm" + MODE: "conv" + CLS_EMBED_ON: True + DIM_MUL: [[1, 2.0], [3, 2.0], [14, 2.0]] + HEAD_MUL: [[1, 2.0], [3, 2.0], [14, 2.0]] + POOL_KVQ_KERNEL: [3, 3, 3] + POOL_KV_STRIDE_ADAPTIVE: [1, 8, 8] + POOL_Q_STRIDE: [[0, 1, 1, 1], [1, 1, 2, 2], [2, 1, 1, 1], [3, 1, 2, 2], [4, 1, 1, 1], [5, 1, 1, 1], [6, 1, 1, 1], [7, 1, 1, 1], [8, 1, 1, 1], [9, 1, 1, 1], [10, 1, 1, 1], [11, 1, 1, 1], [12, 1, 1, 1], [13, 1, 1, 1], [14, 1, 2, 2], [15, 1, 1, 1]] + DROPOUT_RATE: 0.0 + DIM_MUL_IN_ATT: True + RESIDUAL_POOLING: True +AUG: + NUM_SAMPLE: 1 + ENABLE: True + COLOR_JITTER: 0.4 + AA_TYPE: rand-m7-n4-mstd0.5-inc1 + INTERPOLATION: bicubic + RE_PROB: 0.25 + RE_MODE: pixel + RE_COUNT: 1 + RE_SPLIT: False +MIXUP: + ENABLE: True + ALPHA: 0.8 + CUTMIX_ALPHA: 1.0 + PROB: 1.0 + SWITCH_PROB: 0.5 + LABEL_SMOOTH_VALUE: 0.1 +SOLVER: + ZERO_WD_1D_PARAM: True + BASE_LR_SCALE_NUM_SHARDS: True + BASE_LR: 0.0025 + COSINE_AFTER_WARMUP: True + COSINE_END_LR: 1e-6 + WARMUP_START_LR: 1e-4 + WARMUP_EPOCHS: 3.0 + LR_POLICY: cosine + MAX_EPOCH: 100 + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-4 + OPTIMIZING_METHOD: sgd + COSINE_AFTER_WARMUP: True +MODEL: + NUM_CLASSES: 174 + ARCH: mvit + MODEL_NAME: MViT + LOSS_FUNC: soft_cross_entropy + DROPOUT_RATE: 0.5 +TEST: + ENABLE: True + DATASET: ssv2 + BATCH_SIZE: 64 + NUM_ENSEMBLE_VIEWS: 1 + NUM_SPATIAL_CROPS: 3 +DATA_LOADER: + NUM_WORKERS: 2 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . +PREFETCH: + NUM_LOADERS: 3 diff --git a/projects/mvitv2/README.md b/projects/mvitv2/README.md index ad110648e..6844df3ca 100644 --- a/projects/mvitv2/README.md +++ b/projects/mvitv2/README.md @@ -30,6 +30,16 @@ MViT is a multiscale transformer which serves as a general vision backbone for d | MViTv2-L | 40 x 3 | 86.1 | 97.0 | 2828 x 3 x 5 | 217.6 | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/model_zoo/mvitv2/pysf_video_models/MViTv2_L_40x3_k400_f306903192.pyth) | Kinetics/MVITv2_L_40x3_test | +### SSv2 + + +| name | pretrain | frame length x sample rate | top1 | top5 | Flops (G) x views | #params (M) | model | config | +| ------------- | ------------- | ------------- | ------------- | ------------- | ------------- | ------------- | ------------- | ------------- | +| MViTv2-S | K400 | 16 x 4 | 68.2 | 91.4 | 64 x 3 x 1 | 34.4 | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/model_zoo/mvitv2/pysf_video_models/MViTv2_S_16x4_ssv2_f308341823.pyth) | SSv2/MVITv2_S_16x4 | +| MViTv2-B | K400 | 32 x 3 | 70.5 | 92.7 | 225 x 3 x 1 | 51.1 | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/model_zoo/mvitv2/pysf_video_models/MViTv2_B_32x3_ssv2_f305803282.pyth) | SSv2/MVITv2_B_32x3 | +| MViTv2-L | IN21K + K400 | 40 x 3 | 73.3 | 94.1 | 2828 x 3 x 1 | 213.1 | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/model_zoo/mvitv2/pysf_video_models/MViTv2_L_40x3_ssv2_f309603968.pyth) | SSv2/MVITv2_L_40x3 | + + ### ImageNet-1K | name | resolution |acc@1 | #params | FLOPs | 1k model | diff --git a/slowfast/datasets/ssv2.py b/slowfast/datasets/ssv2.py index e5715d69d..11fa6873a 100644 --- a/slowfast/datasets/ssv2.py +++ b/slowfast/datasets/ssv2.py @@ -70,6 +70,16 @@ def __init__(self, cfg, mode, num_retries=10): logger.info("Constructing Something-Something V2 {}...".format(mode)) self._construct_loader() + self.aug = False + self.rand_erase = False + self.use_temporal_gradient = False + self.temporal_gradient_rate = 0.0 + + if self.mode == "train" and self.cfg.AUG.ENABLE: + self.aug = True + if self.cfg.AUG.RE_PROB > 0: + self.rand_erase = True + def _construct_loader(self): """ Construct the video loader. @@ -249,23 +259,58 @@ def __getitem__(self, index): ) ) - # Perform color normalization. - frames = utils.tensor_normalize( - frames, self.cfg.DATA.MEAN, self.cfg.DATA.STD - ) + if self.aug: + if self.cfg.AUG.NUM_SAMPLE > 1: - # T H W C -> C T H W. - frames = frames.permute(3, 0, 1, 2) - # Perform data augmentation. - frames = utils.spatial_sampling( - frames, - spatial_idx=spatial_sample_index, - min_scale=min_scale, - max_scale=max_scale, - crop_size=crop_size, - random_horizontal_flip=self.cfg.DATA.RANDOM_FLIP, - inverse_uniform_sampling=self.cfg.DATA.INV_UNIFORM_SAMPLE, - ) + frame_list = [] + label_list = [] + index_list = [] + for _ in range(self.cfg.AUG.NUM_SAMPLE): + new_frames = utils.aug_frame( + self.cfg, + self.mode, + self.rand_erase, + frames, + spatial_sample_index, + min_scale, + max_scale, + crop_size, + ) + new_frames = utils.pack_pathway_output(self.cfg, new_frames) + frame_list.append(new_frames) + label_list.append(label) + index_list.append(index) + return frame_list, label_list, index_list, [0] * self.cfg.AUG.NUM_SAMPLE, {} + + else: + frames = utils.aug_frame( + self.cfg, + self.mode, + self.rand_erase, + frames, + spatial_sample_index, + min_scale, + max_scale, + crop_size, + ) + else: + # Perform color normalization. + frames = utils.tensor_normalize( + frames, self.cfg.DATA.MEAN, self.cfg.DATA.STD + ) + + # T H W C -> C T H W. + frames = frames.permute(3, 0, 1, 2) + # Perform data augmentation. + frames = utils.spatial_sampling( + frames, + spatial_idx=spatial_sample_index, + min_scale=min_scale, + max_scale=max_scale, + crop_size=crop_size, + random_horizontal_flip=self.cfg.DATA.RANDOM_FLIP, + inverse_uniform_sampling=self.cfg.DATA.INV_UNIFORM_SAMPLE, + ) frames = utils.pack_pathway_output(self.cfg, frames) return frames, label, index, 0, {} diff --git a/slowfast/datasets/utils.py b/slowfast/datasets/utils.py index df8222975..ffd4efd7d 100644 --- a/slowfast/datasets/utils.py +++ b/slowfast/datasets/utils.py @@ -10,10 +10,16 @@ import torch from torch.utils.data.distributed import DistributedSampler +from torchvision import transforms + from slowfast.utils.env import pathmgr from . import transform as transform +from .random_erasing import RandomErasing +from .transform import create_random_augment + + logger = logging.getLogger(__name__) @@ -348,3 +354,99 @@ def loader_worker_init_fn(dataset): dataset (torch.utils.data.Dataset): the given dataset. """ return None + + +def aug_frame( + cfg, + mode, + rand_erase, + frames, + spatial_sample_index, + min_scale, + max_scale, + crop_size, +): + """ + Perform augmentations on the given video frames, including + random augmentation, normalization, spatial sampling and optional random + erasing. + Args: + cfg (CfgNode): configs. + mode (string): Options includes `train`, `val`, or `test` mode. + rand_erase (bool): if performing random erasing. + frames (tensor): frames of images sampled from the video. The + dimension is `num frames` x `height` x `width` x `channel`. + spatial_sample_index (int): if -1, perform random spatial sampling. + If 0, 1, or 2, perform left, center, right crop if width is larger + thanheight, and perform top, center, buttom crop if height is larger + than width. + min_scale (int): the minimal size of scaling. + max_scale (int): the maximal size of scaling. + crop_size (int): the size of height and width used to crop the + frames. + Returns: + frames (tensor): spatially sampled frames. + """ + if cfg.AUG.AA_TYPE: + aug_transform = create_random_augment( + input_size=(frames.size(1), frames.size(2)), + auto_augment=cfg.AUG.AA_TYPE, + interpolation=cfg.AUG.INTERPOLATION, + ) + # T H W C -> T C H W. + frames = frames.permute(0, 3, 1, 2) + list_img = _frame_to_list_img(frames) + list_img = aug_transform(list_img) + frames = _list_img_to_frames(list_img) + frames = frames.permute(0, 2, 3, 1) + + frames = tensor_normalize(frames, cfg.DATA.MEAN, cfg.DATA.STD) + # T H W C -> C T H W. + frames = frames.permute(3, 0, 1, 2) + # Perform data augmentation. + scl, asp = ( + cfg.DATA.TRAIN_JITTER_SCALES_RELATIVE, + cfg.DATA.TRAIN_JITTER_ASPECT_RELATIVE, + ) + relative_scales = None if (mode not in ["train"] or len(scl) == 0) else scl + relative_aspect = None if (mode not in ["train"] or len(asp) == 0) else asp + frames = spatial_sampling( + frames, + spatial_idx=spatial_sample_index, + min_scale=min_scale, + max_scale=max_scale, + crop_size=crop_size, + random_horizontal_flip=cfg.DATA.RANDOM_FLIP, + inverse_uniform_sampling=cfg.DATA.INV_UNIFORM_SAMPLE, + aspect_ratio=relative_aspect, + scale=relative_scales, + motion_shift=cfg.DATA.TRAIN_JITTER_MOTION_SHIFT + if mode in ["train"] + else False, + ) + + if rand_erase: + erase_transform = RandomErasing( + cfg.AUG.RE_PROB, + mode=cfg.AUG.RE_MODE, + max_count=cfg.AUG.RE_COUNT, + num_splits=cfg.AUG.RE_COUNT, + device="cpu", + ) + frames = frames.permute(1, 0, 2, 3) + frames = erase_transform(frames) + frames = frames.permute(1, 0, 2, 3) + + return frames + + +def _frame_to_list_img(frames): + img_list = [ + transforms.ToPILImage()(frames[i]) for i in range(frames.size(0)) + ] + return img_list + + +def _list_img_to_frames(img_list): + img_list = [transforms.ToTensor()(img) for img in img_list] + return torch.stack(img_list) diff --git a/tools/train_net.py b/tools/train_net.py index b4707a4d1..8786d8ebe 100644 --- a/tools/train_net.py +++ b/tools/train_net.py @@ -582,7 +582,7 @@ def train(cfg): convert_from_caffe2=cfg.TRAIN.CHECKPOINT_TYPE == "caffe2", epoch_reset=cfg.TRAIN.CHECKPOINT_EPOCH_RESET, clear_name_pattern=cfg.TRAIN.CHECKPOINT_CLEAR_NAME_PATTERN, - IN_INIT=cfg.TRAIN.CHECKPOINT_IN_INIT, + image_init=cfg.TRAIN.CHECKPOINT_IN_INIT, ) start_epoch = checkpoint_epoch + 1 else: From 0df1057dc68158d2fea25e1812a7549529dcd28b Mon Sep 17 00:00:00 2001 From: Yanghao Li Date: Fri, 19 Aug 2022 22:27:58 -0700 Subject: [PATCH 06/34] set epoch number for PrefetcherDistributedSampler to reduce searching time Summary: Prefetcher sampler needs to search for the matched epoch with the training sampler, which takes a long time when epoch number is large. For example, epoch 766 takes ~17min for this searching in f362812211. Reviewed By: feichtenhofer Differential Revision: D38508933 fbshipit-source-id: 1908c774c7a76c5a69f51eae45b4cb5b862f190d --- slowfast/datasets/loader.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/slowfast/datasets/loader.py b/slowfast/datasets/loader.py index 2b06e8e18..f72552848 100644 --- a/slowfast/datasets/loader.py +++ b/slowfast/datasets/loader.py @@ -206,3 +206,10 @@ def shuffle_dataset(loader, cur_epoch): if isinstance(sampler, DistributedSampler): # DistributedSampler shuffles data based on epoch sampler.set_epoch(cur_epoch) + + if hasattr(loader.dataset, "prefetcher"): + sampler = loader.dataset.prefetcher.sampler + if isinstance(sampler, DistributedSampler): + # DistributedSampler shuffles data based on epoch + print("prefetcher sampler") + sampler.set_epoch(cur_epoch) From 4e9a5b9a34db2de39b2e0e26b40132622db21d88 Mon Sep 17 00:00:00 2001 From: Christoph Feichtenhofer Date: Tue, 23 Aug 2022 10:13:25 -0700 Subject: [PATCH 07/34] imageNet fix Reviewed By: lyttonhao Differential Revision: D38884764 fbshipit-source-id: 89ba2fb698db86b1eec98699f62d7aaa4019f914 --- slowfast/config/defaults.py | 3 +++ slowfast/datasets/imagenet.py | 23 ++++++++++++++++------- tools/train_net.py | 2 ++ 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/slowfast/config/defaults.py b/slowfast/config/defaults.py index f2fda219a..cd9f6394d 100644 --- a/slowfast/config/defaults.py +++ b/slowfast/config/defaults.py @@ -776,6 +776,9 @@ # If not None, use IN1k as val split when training in21k _C.DATA.IN22k_VAL_IN1K = "" +# Large resolution models may use different crop ratios +_C.DATA.IN_VAL_CROP_RATIO = 0.875 # 224/256 = 0.875 + # don't use real video for kinetics.py _C.DATA.DUMMY_LOAD = False diff --git a/slowfast/datasets/imagenet.py b/slowfast/datasets/imagenet.py index a2d45ba66..cb3c8f780 100644 --- a/slowfast/datasets/imagenet.py +++ b/slowfast/datasets/imagenet.py @@ -146,13 +146,22 @@ def _prepare_im_tf(self, im_path): ) else: t = [] - size = int((256 / 224) * test_size) - t.append( - transforms_tv.Resize( - size, interpolation=transforms_tv.InterpolationMode.BICUBIC - ), # to maintain same ratio w.r.t. 224 images - ) - t.append(transforms_tv.CenterCrop(test_size)) + if self.cfg.DATA.IN_VAL_CROP_RATIO == 0.0: + t.append( + transforms_tv.Resize((test_size, test_size), + interpolation=transforms_tv.InterpolationMode.BICUBIC), + ) + else: + size = int( + (1.0 / self.cfg.DATA.IN_VAL_CROP_RATIO) * test_size + ) # = 1/0.875 * test_size + t.append( + transforms_tv.Resize( + size, + interpolation=transforms_tv.InterpolationMode.BICUBIC + ), # to maintain same ratio w.r.t. 224 images + ) + t.append(transforms_tv.CenterCrop(test_size)) t.append(transforms_tv.ToTensor()) t.append( transforms_tv.Normalize(self.cfg.DATA.MEAN, self.cfg.DATA.STD) diff --git a/tools/train_net.py b/tools/train_net.py index 8786d8ebe..d194142ea 100644 --- a/tools/train_net.py +++ b/tools/train_net.py @@ -750,6 +750,8 @@ def train(cfg): train_loader, writer, ) + if start_epoch == cfg.SOLVER.MAX_EPOCH: # eval if we loaded the final checkpoint + eval_epoch(val_loader, model, val_meter, start_epoch, cfg, train_loader, writer) if writer is not None: writer.close() result_string = ( From 64f8d7e1f888c789f6e6ef881fc520cc88589cd0 Mon Sep 17 00:00:00 2001 From: Karttikeya Mangalam Date: Wed, 24 Aug 2022 19:22:15 -0700 Subject: [PATCH 08/34] opensourcing revvit (#587) Summary: Pull Request resolved: https://github.com/facebookresearch/SlowFast/pull/587 sanity check: f363576206 -- MViTv2_B : 84.36% f363575449: K400 rev mvit v1 base 16x4 -- 78.4% f363576094: IN rev vit base 81.6% Reviewed By: chaoyuaw, haooooooqi, lyttonhao Differential Revision: D37575580 fbshipit-source-id: c46ea985426e13f7efdf94698ffe41370f6caebb --- MODEL_ZOO.md | 6 +- README.md | 3 + configs/ImageNet/REV_MVIT_B_16_CONV.yaml | 90 +++ configs/ImageNet/REV_VIT_B.yaml | 88 +++ configs/ImageNet/REV_VIT_S.yaml | 84 +++ configs/Kinetics/MVIT_B_16x4_CONV.yaml | 1 + configs/Kinetics/REV_MVIT_B_16x4_CONV.yaml | 102 +++ projects/rev/README.md | 78 +++ projects/rev/teaser.png | Bin 0 -> 209223 bytes slowfast/config/defaults.py | 19 + slowfast/models/common.py | 84 +++ slowfast/models/reversible_mvit.py | 696 +++++++++++++++++++++ slowfast/models/video_model_builder.py | 214 ++++--- slowfast/utils/checkpoint.py | 1 + tools/train_net.py | 4 + 15 files changed, 1395 insertions(+), 75 deletions(-) create mode 100644 configs/ImageNet/REV_MVIT_B_16_CONV.yaml create mode 100644 configs/ImageNet/REV_VIT_B.yaml create mode 100644 configs/ImageNet/REV_VIT_S.yaml create mode 100644 configs/Kinetics/REV_MVIT_B_16x4_CONV.yaml create mode 100644 projects/rev/README.md create mode 100644 projects/rev/teaser.png create mode 100644 slowfast/models/reversible_mvit.py diff --git a/MODEL_ZOO.md b/MODEL_ZOO.md index cc46af5fe..191ed594e 100644 --- a/MODEL_ZOO.md +++ b/MODEL_ZOO.md @@ -12,6 +12,7 @@ | SlowFast | R50 | 3 x 10 | 4 x 16 | 75.6 | 92.0 | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/model_zoo/kinetics400/SLOWFAST_4x16_R50.pkl) | Kinetics/c2/SLOWFAST_4x16_R50 | K400 | | SlowFast | R50 | 3 x 10 | 8 x 8 | 77.0 | 92.6 | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/model_zoo/kinetics400/SLOWFAST_8x8_R50.pkl) | Kinetics/c2/SLOWFAST_8x8_R50 | K400 | | MViTv1 | B-Conv | 1 x 5 | 16 x 4 | 78.4 | 93.5 | [`link`](https://drive.google.com/file/d/194gJinVejq6A1FmySNKQ8vAN5-FOY-QL/view?usp=sharing) | Kinetics/MVIT_B_16x4_CONV | K400 | +| rev-MViT | B-Conv | 1 x 5 | 16 x 4 | 78.4 | 93.4 | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/rev/REV_MVIT_B_16x4.pyth) | Kinetics/REV_MVIT_B_16x4_CONV | K400 | | MViTv1 | B-Conv | 1 x 5 | 32 x 3 | 80.4 | 94.8 | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/model_zoo/mvit/k400.pyth) | Kinetics/MVIT_B_32x3_CONV | K400 | | MViTv1 | B-Conv | 1 x 5 | 32 x 3 | 83.9 | 96.5 | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/model_zoo/mvit/k600.pyth) | Kinetics/MVIT_B_32x3_CONV_K600 | K600 | | MViTv2 | S | 1 x 5 | 16 x 4 | 81.0 | 94.6 | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/model_zoo/mvitv2/pysf_video_models/MViTv2_S_16x4_k400_f302660347.pyth) | Kinetics/MVITv2_S_16x4 | K400 | @@ -70,8 +71,11 @@ We also release the imagenet pretrained model if finetuning from ImageNet is pre | ------------- | ------------- | ------------- | ------------- | ------------- | ------------- | | ResNet | R50 | 76.4 | 93.2 | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/model_zoo/kinetics400/R50_IN1K.pyth) | ImageNet/RES_R50 | | MVIT | B-16-Conv | 82.9 | 96.3 | [`link`](https://drive.google.com/file/d/1dYYqUB-3DSgBVc9d6o-rW8ojtVsrFLgp/view?usp=sharing) | ImageNet/MVIT_B_16_CONV | +| rev-VIT | Small | 79.9 | 94.9 | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/rev/REV_VIT_S.pyth) | ImageNet/REV_VIT_S.yaml | +| rev-VIT | Base | 81.8 | 95.6 | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/rev/REV_VIT_B.pyth) | ImageNet/REV_VIT_B.yaml | +| rev-MVIT | Base | 82.9* | 96.3 | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/rev/REV_MVIT_B.pyth) | ImageNet/REV_MVIT_B_16_CONV.yaml | - +*please refer to [Reversible Model Zoo](projects/rev/README.md). ## PyTorchVideo diff --git a/README.md b/README.md index 53a03c6b9..25093efad 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@ PySlowFast is an open source video understanding codebase from FAIR that provide - [MViTv2: Improved Multiscale Vision Transformers for Classification and Detection](https://arxiv.org/abs/2112.01526) - [Masked Feature Prediction for Self-Supervised Visual Pre-Training](https://arxiv.org/abs/2112.09133) - [Masked Autoencoders As Spatiotemporal Learners](https://arxiv.org/abs/2205.09113) +- [Reversible Vision Transformers](https://openaccess.thecvf.com/content/CVPR2022/papers/Mangalam_Reversible_Vision_Transformers_CVPR_2022_paper.pdf)
@@ -27,8 +28,10 @@ The goal of PySlowFast is to provide a high-performance, light-weight pytorch co - Non-local Network - X3D - MViTv1 and MViTv2 +- Rev-ViT and Rev-MViT ## Updates + - We now [Reversible Vision Transformers](https://openaccess.thecvf.com/content/CVPR2022/papers/Mangalam_Reversible_Vision_Transformers_CVPR_2022_paper.pdf). Both Reversible ViT and MViT models released. See [`projects/rev`](./projects/rev/README.md). - We now support [MAE for Video](https://arxiv.org/abs/2104.11227.pdf). See [`projects/mae`](./projects/mae/README.md) for more information. - We now support [MaskFeat](https://arxiv.org/abs/2112.09133). See [`projects/maskfeat`](./projects/maskfeat/README.md) for more information. - We now support [MViTv2](https://arxiv.org/abs/2104.11227.pdf) in PySlowFast. See [`projects/mvitv2`](./projects/mvitv2/README.md) for more information. diff --git a/configs/ImageNet/REV_MVIT_B_16_CONV.yaml b/configs/ImageNet/REV_MVIT_B_16_CONV.yaml new file mode 100644 index 000000000..0572d92af --- /dev/null +++ b/configs/ImageNet/REV_MVIT_B_16_CONV.yaml @@ -0,0 +1,90 @@ +TRAIN: + ENABLE: True + DATASET: imagenet + BATCH_SIZE: 256 + EVAL_PERIOD: 10 + CHECKPOINT_PERIOD: 1 + AUTO_RESUME: True + +DATA: + # PATH_TO_DATA_DIR: path-to-imagenet-dir + MEAN: [0.485, 0.456, 0.406] + STD: [0.229, 0.224, 0.225] + NUM_FRAMES: 1 + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 224 + INPUT_CHANNEL_NUM: [3] +MVIT: + PATCH_2D: True + ZERO_DECAY_POS_CLS: False + MODE: "conv" + CLS_EMBED_ON: False + PATCH_KERNEL: [7, 7] + PATCH_STRIDE: [4, 4] + PATCH_PADDING: [3, 3] + EMBED_DIM: 96 + NUM_HEADS: 1 + MLP_RATIO: 4.0 + QKV_BIAS: True + DROPPATH_RATE: 0.1 + DEPTH: 16 + NORM: "layernorm" + DIM_MUL: [[1, 2.0], [3, 2.0], [14, 2.0]] + HEAD_MUL: [[1, 2.0], [3, 2.0], [14, 2.0]] + POOL_KVQ_KERNEL: [1, 3, 3] + POOL_KV_STRIDE_ADAPTIVE: [1, 4, 4] + POOL_Q_STRIDE: [[1, 1, 2, 2], [3, 1, 2, 2], [14, 1, 2, 2]] + SEPARATE_QKV : True + REV: + ENABLE: True + RESPATH_FUSE: "concat" + BUFFER_LAYERS : [1,3, 14] + RES_PATH : "conv" + PRE_Q_FUSION: "concat_linear_2" +AUG: + ENABLE: True + COLOR_JITTER: 0.4 + AA_TYPE: rand-m9-n6-mstd0.5-inc1 + INTERPOLATION: bicubic + RE_PROB: 0.25 + RE_MODE: pixel + RE_COUNT: 1 + RE_SPLIT: False +MIXUP: + ENABLE: True + ALPHA: 0.8 + CUTMIX_ALPHA: 1.0 + PROB: 1.0 + SWITCH_PROB: 0.5 + LABEL_SMOOTH_VALUE: 0.1 +SOLVER: + BASE_LR_SCALE_NUM_SHARDS: True + BASE_LR: 0.00025 + LR_POLICY: cosine + MAX_EPOCH: 300 + MOMENTUM: 0.9 + WEIGHT_DECAY: 0.05 + WARMUP_EPOCHS: 70.0 + WARMUP_START_LR: 1e-8 + OPTIMIZING_METHOD: adamw + COSINE_AFTER_WARMUP: True + COSINE_END_LR: 1e-6 + ZERO_WD_1D_PARAM: True + CLIP_GRAD_L2NORM: 1.0 +MODEL: + NUM_CLASSES: 1000 + ARCH: mvit + MODEL_NAME: MViT + LOSS_FUNC: soft_cross_entropy + DROPOUT_RATE: 0.0 +TEST: + ENABLE: False + DATASET: imagenet + BATCH_SIZE: 256 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git a/configs/ImageNet/REV_VIT_B.yaml b/configs/ImageNet/REV_VIT_B.yaml new file mode 100644 index 000000000..953dff573 --- /dev/null +++ b/configs/ImageNet/REV_VIT_B.yaml @@ -0,0 +1,88 @@ +TRAIN: + ENABLE: True + DATASET: imagenet + BATCH_SIZE: 512 + EVAL_PERIOD: 10 + CHECKPOINT_PERIOD: 1 + AUTO_RESUME: True + +DATA: + # PATH_TO_DATA_DIR: path-to-imagenet-dir + MEAN: [0.485, 0.456, 0.406] + STD: [0.229, 0.224, 0.225] + NUM_FRAMES: 1 + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 224 + INPUT_CHANNEL_NUM: [3] +MVIT: + # USE_SOFTMAX: False + PATCH_2D: True + ZERO_DECAY_POS_CLS: False + MODE: "conv" + CLS_EMBED_ON: False + PATCH_KERNEL: [16, 16] + PATCH_STRIDE: [16, 16] + PATCH_PADDING: [0, 0] + EMBED_DIM: 768 + NUM_HEADS: 12 + MLP_RATIO: 4.0 + QKV_BIAS: True + DROPPATH_RATE: 0.2 # 0.1 + DEPTH: 12 + NORM: "layernorm" + POOL_KV_STRIDE: [] + SEPARATE_QKV : True + REV: + ENABLE: True + RESPATH_FUSE: "concat" + RES_PATH: "conv" + PRE_Q_FUSION: "concat" + +AUG: + ENABLE: True + COLOR_JITTER: 0.4 + AA_TYPE: rand-m6-n5-mstd0.5-inc1 + INTERPOLATION: bicubic + RE_PROB: 0.25 + RE_MODE: pixel + RE_COUNT: 1 + RE_SPLIT: False +MIXUP: + ENABLE: True + ALPHA: 0.8 + CUTMIX_ALPHA: 1.0 + PROB: 1.0 + SWITCH_PROB: 0.5 + LABEL_SMOOTH_VALUE: 0.1 +SOLVER: + BASE_LR_SCALE_NUM_SHARDS: True + BASE_LR: 0.0005 + # BASE_LR: 0.0003 + LR_POLICY: cosine + MAX_EPOCH: 300 + MOMENTUM: 0.9 + WEIGHT_DECAY: 0.1 + WARMUP_EPOCHS: 70.0 + WARMUP_START_LR: 1e-8 + OPTIMIZING_METHOD: adamw + COSINE_AFTER_WARMUP: True + COSINE_END_LR: 1e-6 + ZERO_WD_1D_PARAM: True + CLIP_GRAD_L2NORM: 2.0 +MODEL: + NUM_CLASSES: 1000 + ARCH: mvit + MODEL_NAME: MViT + LOSS_FUNC: soft_cross_entropy + DROPOUT_RATE: 0.0 +TEST: + ENABLE: False + DATASET: imagenet + BATCH_SIZE: 256 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git a/configs/ImageNet/REV_VIT_S.yaml b/configs/ImageNet/REV_VIT_S.yaml new file mode 100644 index 000000000..86d867369 --- /dev/null +++ b/configs/ImageNet/REV_VIT_S.yaml @@ -0,0 +1,84 @@ +TRAIN: + ENABLE: True + DATASET: imagenet + BATCH_SIZE: 512 + EVAL_PERIOD: 10 + CHECKPOINT_PERIOD: 1 + AUTO_RESUME: True + +DATA: + # PATH_TO_DATA_DIR: path-to-imagenet-dir + MEAN: [0.485, 0.456, 0.406] + STD: [0.229, 0.224, 0.225] + NUM_FRAMES: 1 + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 224 + INPUT_CHANNEL_NUM: [3] +MVIT: + PATCH_2D: True + ZERO_DECAY_POS_CLS: False + MODE: "conv" + CLS_EMBED_ON: False + PATCH_KERNEL: [16, 16] + PATCH_STRIDE: [16, 16] + PATCH_PADDING: [0, 0] + EMBED_DIM: 384 + NUM_HEADS: 6 + MLP_RATIO: 4.0 + QKV_BIAS: True + DROPPATH_RATE: 0.1 + DEPTH: 12 + NORM: "layernorm" + POOL_KV_STRIDE: [] + SEPARATE_QKV : True + REV: + ENABLE: True + RESPATH_FUSE: "concat" + RES_PATH: "conv" +AUG: + ENABLE: True + COLOR_JITTER: 0.4 + AA_TYPE: rand-m7-mstd0.5-inc1 + INTERPOLATION: bicubic + RE_PROB: 0.25 + RE_MODE: pixel + RE_COUNT: 1 + RE_SPLIT: False +MIXUP: + ENABLE: True + ALPHA: 0.8 + CUTMIX_ALPHA: 1.0 + PROB: 1.0 + SWITCH_PROB: 0.5 + LABEL_SMOOTH_VALUE: 0.1 +SOLVER: + BASE_LR_SCALE_NUM_SHARDS: True + BASE_LR: 0.0005 + LR_POLICY: cosine + MAX_EPOCH: 300 + MOMENTUM: 0.9 + WEIGHT_DECAY: 0.05 + WARMUP_EPOCHS: 70.0 + WARMUP_START_LR: 1e-8 + OPTIMIZING_METHOD: adamw + COSINE_AFTER_WARMUP: True + COSINE_END_LR: 1e-6 + ZERO_WD_1D_PARAM: True + CLIP_GRAD_L2NORM: 1.0 # this is commented now +MODEL: + NUM_CLASSES: 1000 + ARCH: mvit + MODEL_NAME: MViT + LOSS_FUNC: soft_cross_entropy + DROPOUT_RATE: 0.0 +TEST: + ENABLE: False + DATASET: imagenet + BATCH_SIZE: 256 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git a/configs/Kinetics/MVIT_B_16x4_CONV.yaml b/configs/Kinetics/MVIT_B_16x4_CONV.yaml index b5faead29..fe6a49f55 100644 --- a/configs/Kinetics/MVIT_B_16x4_CONV.yaml +++ b/configs/Kinetics/MVIT_B_16x4_CONV.yaml @@ -79,6 +79,7 @@ MODEL: LOSS_FUNC: soft_cross_entropy DROPOUT_RATE: 0.5 TEST: + ENABLE: True DATASET: kinetics BATCH_SIZE: 64 diff --git a/configs/Kinetics/REV_MVIT_B_16x4_CONV.yaml b/configs/Kinetics/REV_MVIT_B_16x4_CONV.yaml new file mode 100644 index 000000000..af2c72805 --- /dev/null +++ b/configs/Kinetics/REV_MVIT_B_16x4_CONV.yaml @@ -0,0 +1,102 @@ +TRAIN: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 16 + EVAL_PERIOD: 10 + CHECKPOINT_PERIOD: 10 + AUTO_RESUME: True + +DATA: + DECODING_BACKEND: torchvision + NUM_FRAMES: 16 + SAMPLING_RATE: 4 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 224 + INPUT_CHANNEL_NUM: [3] + # PATH_TO_DATA_DIR: path-to-k400-dir + TRAIN_JITTER_SCALES_RELATIVE: [0.08, 1.0] + TRAIN_JITTER_ASPECT_RELATIVE: [0.75, 1.3333] + +MVIT: + ZERO_DECAY_POS_CLS: False + CLS_EMBED_ON: True + SEP_POS_EMBED: True + USE_ABS_POS: True + DEPTH: 16 + NUM_HEADS: 1 + EMBED_DIM: 96 + PATCH_KERNEL: (3, 7, 7) + PATCH_STRIDE: (2, 4, 4) + PATCH_PADDING: (1, 3, 3) + MLP_RATIO: 4.0 + QKV_BIAS: False + DROPPATH_RATE: 0.05 + NORM: "layernorm" + MODE: "conv" + DIM_MUL: [[1, 2.0], [3, 2.0], [14, 2.0]] + HEAD_MUL: [[1, 2.0], [3, 2.0], [14, 2.0]] + POOL_KVQ_KERNEL: [3, 3, 3] + POOL_KV_STRIDE_ADAPTIVE: [1, 8, 8] + POOL_Q_STRIDE: [[1, 1, 2, 2], [3, 1, 2, 2], [14, 1, 2, 2]] + + REV: + ENABLE: True + RESPATH_FUSE: "concat" + BUFFER_LAYERS : [1, 3, 14] + RES_PATH : "conv" + +AUG: + ENABLE: True + COLOR_JITTER: 0.4 + AA_TYPE: rand-m7-n4-mstd0.5-inc1 + INTERPOLATION: bicubic + NUM_SAMPLE: 2 + RE_MODE: pixel + RE_COUNT: 1 + RE_SPLIT: False + +MIXUP: + ENABLE: True + ALPHA: 0.8 + LABEL_SMOOTH_VALUE: 0.1 + CUTMIX_ALPHA: 1.0 + PROB: 1.0 + SWITCH_PROB: 0.5 + +SOLVER: + BASE_LR_SCALE_NUM_SHARDS: True + BASE_LR: 0.0001 + LR_POLICY: cosine + COSINE_AFTER_WARMUP: True + COSINE_END_LR: 1e-6 + MAX_EPOCH: 200 + WARMUP_EPOCHS: 50.0 + MOMENTUM: 0.9 + WEIGHT_DECAY: 7e-2 + ZERO_WD_1D_PARAM: True + WARMUP_START_LR: 0.000001 + OPTIMIZING_METHOD: adamw + +MODEL: + NUM_CLASSES: 400 + ARCH: slow + MODEL_NAME: MViT + LOSS_FUNC: soft_cross_entropy + DROPOUT_RATE: 0.5 + +TEST: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 64 + NUM_SPATIAL_CROPS: 1 + NUM_ENSEMBLE_VIEWS: 5 # [1, 3, 5, 7, 10] + +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True + +NUM_GPUS: 8 +NUM_SHARDS: 16 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git a/projects/rev/README.md b/projects/rev/README.md new file mode 100644 index 000000000..bdc8f0848 --- /dev/null +++ b/projects/rev/README.md @@ -0,0 +1,78 @@ +# Reversible Vision Transformers + +Official PyTorch implementation of **Rev-ViT** and **Rev-MViT** models, from the following paper: +
[Reversible Vision Transformers](https://openaccess.thecvf.com/content/CVPR2022/papers/Mangalam_Reversible_Vision_Transformers_CVPR_2022_paper.pdf) +[Karttikeya Mangalam](https://karttikeya.github.io/)\*, [Haoqi Fan](https://haoqifan.github.io/)\, [Yanghao Li](https://lyttonhao.github.io/)\, [Chao-Yuan Wu](https://chaoyuan.org/)\, [Bo Xiong](https://www.cs.utexas.edu/~bxiong/)\, [Christoph Feichtenhofer](http://feichtenhofer.github.io/)\*, [Jitendra Malik](http://people.eecs.berkeley.edu/~malik/) +CVPR 2022 (Oral) + +**Project Homepage**: https://karttikeya.github.io/publication/revvit/ + +
+
+ +
+
+ +## Pretrained Models + +### ImageNet + +| Architecture | #params (M) | FLOPs (G) | Top1 | weights | Config | +| ------------ | ----------- | --------- | ---- | ------- | --------------------- | +| Rev-ViT-S | 22 | 4.6 | 79.9 | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/rev/REV_VIT_S.pyth) | ImageNet/REV_VIT_S | +| Rev-ViT-B | 87 | 17.6 | 81.8 | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/rev/REV_VIT_B.pyth) | ImageNet/REV_VIT_B | +| Rev-MViT-B | 39 | 8.7 | 82.9* | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/rev/REV_MVIT_B.pyth) | ImageNet/REV_MVIT_B_16_CONV | + +*improved from 82.5% reported in the [Paper Table 1](https://openaccess.thecvf.com/content/CVPR2022/papers/Mangalam_Reversible_Vision_Transformers_CVPR_2022_paper.pdf). + +### Kinetics 400 + +| Architecture | frame length x sample rate | Top1 | Top5 | Flops (G) x views | #params (M) | weights | config | +| ------------ | -------------------------- | ---- | ---- | ----------------- | ----------- | ------- | --------------------------------- | +| Rev-MViT-B | 16 x 4 | 78.4 | 93.4 | 64 x 1 x 5 | 34.9 | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/rev/REV_MVIT_B_16x4.pyth) | Kinetics/REV_MVIT_B_16x4_CONV | + +## Getting started + +To use Rev-ViT (or Rev-MViT) image models please refer to the configs under `configs/ImageNet`, and see [paper](https://openaccess.thecvf.com/content/CVPR2022/papers/Mangalam_Reversible_Vision_Transformers_CVPR_2022_paper.pdf) for details. For example, the command + +``` +python tools/run_net.py \ + +--cfg configs/ImageNet/REV_VIT_B.yaml \ + +DATA.PATH_TO_DATA_DIR path_to_your_dataset \ +NUM_GPUS 1 \ +``` + +should train and test a Reversible ViT Base Image model (trained with DeiT recipe) on your dataset. Please refer to [general repo-level](https://github.com/facebookresearch/SlowFast/blob/main/GETTING_STARTED.md) instruction for futher details. + +For Rev-MViT video models, please run the `configs/Kinetics` configs as, +``` + +python tools/run_net.py \ + +--cfg configs/Kinetics/REV_MVIT_B_16x4_CONV.yaml \ + +DATA.PATH_TO_DATA_DIR path_to_your_dataset \ +NUM_GPUS 1 \ +``` + +## Citing Rev-ViT \& Rev-MViT + +If you find Reversible Models useful for your research, please consider citing the paper using the following BibTeX entry. + +```BibTeX + + +@inproceedings{mangalam2022, + + title = {Reversible Vision Transformers}, + + author = {Mangalam, Karttikeya and Fan, Haoqi and Li, Yanghao and Wu, Chao-Yuan and Xiong, Bo and Feichtenhofer, Christoph and Malik, Jitendra}, + + booktitle = {Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition}, + + year = {2022}, +} + +``` diff --git a/projects/rev/teaser.png b/projects/rev/teaser.png new file mode 100644 index 0000000000000000000000000000000000000000..39abdfd83eb2b6c649be0004f9ea6d4c1835c7fc GIT binary patch literal 209223 zcmd42Wmp{D(l$B_gS!*l-QC?S1a}Bd(BQ7YLlPW<2MNK0ySqz(;O-jSInA@5{k-ov z-*@f(_srFEO;2~P)vIe&t-7n~?nqS?SrkM9L=XssA}=SU1_HqngFw(V@UXxU?;tAR z0OGohq@=36q$HWDv!kVrodpOa7nzt0r=hM(5P0IVM@}v#0m=KYoL2`@lR(S9U|=9e zph`f;3Jykz?Z6!FUM-9W&ZQ*FjfgOn{`r>li@3Nh91@Sl6_jF1s&JFfRlrW` zea7?SHS6`F&*(FV49=x{-Bkx65cy3K)#6Gt1|i;9yD&7L7}(qdrkda2M+y!B0Wx}O z>NCU13E0}X)q;KRV5RL@`_0;)d;?H6Nab9I#-W%K7Crhlj8>zPyRNue0^K?OVB^eiztii zcK#x?gEO7!!ffc#{3oSV3hQM|%3FEMdLgaK?lkT$q5(StUA+VY*43)5Cl{DlcFuI> zk4Ai5lhqN~x`d7mMifZNJ7ajaJ|n%C@w1uakLoQFwZ8UAWE^5ff>t~Mtr6S@>I95q zF@4v6GHN2@7NV<~H-C2Tcg;-67#Xl+}Uzo_UAeuGHQhDy6w?N_O4;=2y#y zUHhuWVZZ)2{+}09<8%$xUV%nJL50F8)2|Mp0@Xlb>TudL1DP{9w8r>i5Nt8%eKEpa zXe^T;^Bs7DAlkV1Wbjb2WT;F~24tXDV&J3@O%q}|NRJ8QZ&Zd3BUL2SV9Hf|YuNAY zx=v6AP`vHLr|1_!;-{D&gRf19g2{-a!V13#3QD8xTZMZzjaQ5DUVbU!>Shj-ixA12;}pah1nLyHZ|vP0a7su^D; zD7mA-399*3S)lD|mlKvRZYy{t=zcZ%6dy4gTh%pzWf0sW<&`U;`lrx50a632`GG-p zI-5YXw2|Fq`B=a$Ytih>usv0CXATAxwhRnJ+URxiFN+YB~>bBw<3>f4aM@%80@LT^P0 zz>1$N8FAqlU9VHRZ~@U zT-`|vULBoblU{?4NVC3hy!P)|tu#s?(fI%G`7aLcHDQgp> zmN+S&ZxMzbY3n4A!<;lv=r(8?f`2u{>XbYXzjz}C##SKd;aRlj~S zrVeML^OZeDSur1H>eG0mO>PlG(N)nz5u--2#_Qs%i8rp40%@5lD-xnIZHmuAvbLzU zr~#wB{3tFCqqo^S+9T>QLPRRWSr+QP~$QZ2fVKVkK{lqi+V8hS}$&Iu@8 zY+MX?oOf_Xgn6!mXi97fUX?YdRcW0UZKovFCbcE$*nXekFU7ELto`m@s>v74Qz5aKdp0=_by(x%^(7FtBc2XtpA;%sdyGX#VQef=h*zBwZ`r zBHc#4#POSBrE=~pF)>K9Su;{Id79T2?!fiHZwAd?!wzhJX0KxnVs1LC6&*QUYsIOTT&mmLc|J~bXqXoGz#-}}EWX!r0A)=geV zW@Q>j8MmwywuL+^JYRv7#8k!P#aQ*DM=WN4%0g#=wc*@hV}mJz8A38)(_j>X_3_$~ zz<750r`9=Z5)+-<=;)oObGT=GzxZ5rcgx@Yt{ATXcYgua2LPB4XMi_;`Dyj?t;jL(F; zESlf861F8g3(YY%aCy=7Fk9#kRE|$qRnri6x0&P6-;5 z8|I@|!^n>m;H=@)HouA(=?PtLwO|>lWhG36$X_Xv@0r>t;-c6jlnpAGuMFH(Emaj) zin4q*^7jorL!5{z=C|Z0bjjSg+j^?TY>8{CG)p`#ax1>muF}#j51OGW-P0*qpcd_LJn!a<@AZ6P$^J$xj2Y&pF@j&UdS zaCp$_W^6RDGc+)Iu)y+|+9I{{FT8nc>T{f%>lMgv!!tI0yJ zy|Tr}%5Rk(3&{>`8V<&-9&qo&?PQImour3ULNfF{{C{9;EdMwwdbgP0V7pO<*5uXX zc>HH^szTe=@z3&@C}Kdf_L1?DhM9)CQFQ4@nO`ldxAE-j;jH~D+e6mn=gYudv6YU~ zkPJ5!H@9#uH2WXn2NsuI;aI3EeCsZG`5oVK8e^Are6fiViT(b#tdDtoh9N%S8x>kU zjb3AIQc-=4%`L9Xubev)yxp2qpOhINL@TIgsQ>!m-R(h;dgJ#ZmEz3hNf|pv>omT2 zAIlQMMGm}V5?i6D1F1e_Gh}@32NtYk=p|dp9n3*VCHK z%kDH~$LSRkPpjuLGbY!=f%}iw8>CSpMFB!Lk+(k%T!wdcHip_(TQz*SALs9wcNcOa zKSm<3%XX%|gHi1djE{%%bOG&s-T-AUg97WV8{0B5t?#l-qS^!2_egvdh@c9OV91=Q zK^?Q8-@1|2)e!-RXU6CX;umG0+N$`^0g+sDqwe!~1JE|gB(Ig4xuHPX;P%g*dVgUBxK~Ui~APBGr1$+sBFMy5X!a#7qFDCGn$_D@YEG%(0^uPC^ zYhErCSC^ER2Y%JfoGmOIT&x{k8~QW1fUah3G;~~bl$8X`9PL@&nmd|Wuz1=#y)*#{ zc?tl#_7<*h$vo}t99#rEg(?2JLIBu*Im}8y_SYq@w!#!T%Bp0Nj?NZjJS=Q1Y!o7h zWMpJQ&gPZ^YEm+PHwXR+Q&_vYItj3{dU$xScyO{fI$N=_^YinwvT?9-a4-W`FuQm; zxW4sdc5tEmS114ON6Ny*%-P1t)yC0*?4{qgrjBl|!W0xQH~Q!EuW?#<+WgO*99;f> zEZ_lIU(T?yv#_!L(>Ksm=;f$@s*R_GovxIPJs=+79wPi){6c?S|9_nMpF94qmOB5_ zlAW7}`+qn6U#I@RO*LICoFyIYfqS}&{Lj?<-S~f>{JWtL>&v76FH!tU&VL;R1TBIn z#QM*qi69R8VTFP~Vjy`baSczX!wduyoayO4G;?YM1avw0dlmS~=-|L_frt@$H4YI` zKDN{EO7#u*jgGX7YaD83@@uCXHJtG9uzO(4S3D#FK?q;I2$TDsw|Rzpos2BRc@5}4 z$&zxq?2Kmk1jILRx{L|$RP&ifsLBOG{>zpf3(?~_^CVFLQ;GeTEee!~`XEULh54U1 zrw_n&pG?fj|7(XD0;i7ub%5;{U=0Dj;MmYFz*GOnO?9um?Y|9<3`!PWb#E1323O+0 zJS~-2KJ+u}f014wgos=$KZFur7yiEt?#qkt2>-L>fV2O<7hboZ8X^*TkPJEd)PK_Hhc8IF2(Icp5(%C*D@^C2?zzs}MFMs|LPsKg?vGniSwalaK4 z3ro^pCG$VtN3LBkCT6H)N5{pAQ8X2l%0XW!8ot;?bHPW@*o`w zRbyk;i1%Q5d3g+mRnxuws#3sH_|qTk?^BVJi>s-tr`=r$ks@>>2kq~_`%zM&aWvB3 z{{@#>uUlzhP8HKy0qpzPPGMe1y98_eB=*mrKXFM(xCeZ{{W0NR1%3TW@T0I$b?f{0 z?{M6!#FYv?CLQI?XHh~ot06^OA+RWjDS-#r=YeQuGXHS`{;~}xYCvBZ(Hvs`qbUJ~ z#`U7HD3HFc|0-Mq1VA){A^QK--chL9#U}sVWxEX+zAQC8y(B6oI@+btGf5UqWkMbZ znuLFvt)$*D)~Rqxgb>jN?xgzH@*D9XQ9)s{A?U17-r8~gbP9z?NRY9!$DYKJlMhMA zwssHSVF|VvuWnHiPXI4aI+laLt>I!AF-#QLLn6bf`y?Na_~>xumKG3HHsheNVrU2bgL}6^%EHx8KvR8-_2~x)p^s{U_nn07W8){lwq+n6?7i$7k zzrvD^fiaHw-lHD7HFJ94ecfwIt{)#gaVmGbk3Ho@)JbOjdeGK5{ zOkM+~@@8^s3g*P+{rmS|Mjj*x2oTnp^oHQ+~1l@ z@v;P9cur814%RKXr>*WNnL`zK`-`C0KS@{-Z z!OC*&EKV0eMbMRAW0rdv(@hAro$CLrQhdsR3UJ>eE1NB;SIeB*$wVx7@(T)Nx`*En zlXaJ5)!Hv8xVyVoRc;QXWrCQ9FAi;oo3{@S3ff?W%x*BpucgvzDRgypW6J1`xffk_ zUHE;PCiry45-`Tca?j43=h_0=oW->OtD|vK{ey0Smo`EkWJwk%VrkqXGO&{7=;Pk) zl|9*ki-PsHa*&w<|6J!$NTr zV+6;n3DQ6-cxEV}v5OYm06n@_L%U5;2W(SLG#}Rs9ID} z2vU<(%u%e8A!#!Ck|7g++MOr^J1A40B!ShBLS-$`JNzCFyf2493}p~7T8~O>oJ2GS zNk1n2VI6VU1~=2X2fw;JG%)a{j1+?t>6kR+jerTd2HFQ6!%{CA%m`s{z4qHBVtUgZ zJ^5K;Ep+|pI)&@uV}VFTOn64H)4lx5XoaC#Wy3Px(~Y(FN9N=WRi-6?*2%1Axgw&` zf1DjpUQ^WUm+-!r>W2iewdT46Hwn4dot=PGh2Txt)Z6qmH8l|u(17I`3ey@MN(r0s z2UD!FbebPYD__8mvByREf8#dvWk{hb{K-MrJd8Mq=2>rYa`NI`H`5=mk-9czR(UF; zpdun=!Ey=RUpM;-y6cm?yi}{cQUCG#C;6}MX19P1@IXp93rkN7cyv>xUivjc5Q}X1 z<&?GgI>}CdofYXi{@-r$13R#cb>6I3yz9Vd&rV2^b4?4vJJl4_Ry^v^(~TQTw)0~) zv-tt5#a@3m#hNrfS{oP03f?6hsx3_lNwiR_bPp;ofjXJGPQ9GNIPDmg1GCGu??06< ztAb$}`jLjL8p6wvggYwWGRm&@{zQ!xEQO$SF8F z>+*#EG&eOK@+~<~i|5coZZ8l|XVG>|gbk}jJApac9NZz4%SNYeWK`xKyJ$I+0qM*D z9V_~khaQO0?dxb6(hy$RA;Z(QMS?rKl}5y}D=XEO9G&>(UZ$P+MMAocotURfV87Y3Y%i@}icu-G)p4J^8VUK>z(M?j>H>80|euosF*${sUiUtvVHD>d@3I3=R3f@-@Ro-f z2CF!y!ZjJ2VnH5VAND{!^IV(bSGBvlPG~WKg7#YFp|I zm<4{?^kUwIzYe{0>=YqpFvM4t=iN1V+E*GqeAHWJM?cR{>`{hLl+reKAFecO*6+lUdmU(?08`AU-zDuVNyH-$%HRq=q7Xv#HS31mk|0w)%O80i4 z)l{_FYbxzaHF0I-b*?R0L;aMh z8X8nXNzCePBWb*{M$kHDnAWD?`<8A;I^DYLNZZb-*uVg=p!*@){Y`K%_!*83?k_86 zhaxs+5foN*5ZR*P|M`5wR#xgajXL|t%hCG$MAX zB!iB^B~iWzz|zhSb? zzkcb_262Tw6gH+J3ZCM|T};gq=}07_R>>D6*5>itNSE#5Scj!XuRI^co6Y#wn+0hO zXTMu~Z=L0pgdU)RJ8}o&X_2t8zoc?n4cgZ&ZNFC{Lr|qMbRM9tbXXEd^xGZJIlDV3 zzZb5~%afu}N#Er)u6T9abRxok(tI8){P9+vj0s-jc$tCw-Z1g=OwX*Fy2k4}YKf%SQaVXZmg1 z@jx}c2$AqB&+%>EK|K^wJx*~>YkoKqwXk1XyI$IMXu7|R$BNtPG#gD`TMGXn&~&oH zgTR#(*t$}Y_u>wdKqpn>XDAkXuI1PYjZA3}Fj>Wm%QJ|syInpxU!_~T9Thx*Oc&Ly z_%Ap<@z~GTBj1dDj=fGbNEC%RwP>x!tcb)7j6^a(|f)}Q(n0yARL7T4b#m`;Xc-}^_C`^!DMx} z3r$qRi=B?waao9eFE1V}F!ObwXfyEs6TNzYj>iiUGru`qV<`1FS>6^w_TP8qoUn&U z!mk(~2S=mT&zw@Hrlz*KyO1J$;0}7U_3N`QWLjk6ruLyCZ}C&Q{tQI^xCbqttQ0LQ z^4m>^_xJ6e9&QzGPv>pRdlqHS_;=FHuCj`?%W-u`Hn+E7AFmycHrBLkeigo9v1rf8 zJRU05^1@FT-(!K(;c+URDKoS!5YwG(bhC9?aG(|zUSdhLc`>DNDhSP%azd-F-&W5p ztKxgz_wngd3-ZVd()?blRH+=h8j+k}S;<1DQ<> zFgnWfm}h1}uaghG=+JzB+_X)vB7CWV7O?NM@gb|pq+mGU>74Z1Jo#vS-Bet-#=z&O z#(6u{YG0W}o~#I71IIpjf1TKWp*rA5VnCqxJoC)Q1lywixbYz2p6Z$==k*<6uv?iv z$LwaC?PqH(*UQoTb+%3+A3Oz2z(L^qs|807*=?7m!a}+2(G1(|+qp6Sf)a~ih}qV+ zXcje5TY{M!^Q+lDR{h%H9vG~$h5lH|rRa5@zgBt}DB%mQ2hYA zq(+PC!7qxR(#Rhx@4ucrji_N{{u73`Q=u=3zzkbD=kp`jt+X0yl^N_^)&V)3?GmH* zowD+BG(N}dS9m+D2z0jDw_pY)3C;|+8C}bfQcmR&sX81w!jjujz76C$qO!CKu69sd z5^L)Xg*7h~{znNrSZ0*>O+nrVSuEiI81?{%@1wzqc~oXTl@MJtD9k)%Yd5%3f?Uk$ zUK{)0THqa2@Gg6HFiz@k3gO26LuPg%AF6tQpsyn#}g4A?GWO}bAHFkEqOydLnYq>95 zaMP;i_H}5oPq+baxNQu@OiXd!D6LVN<++Z~(dU;JJsn^mtsLrGw|6hYZR2<=a_hXL z4VC!QdxyZw=818#)NnTN%xCBqYg3l1&BENmafJ%0sDHE2L&c027d{s|&{o zs)35eW5tE&hkhgx))R2s&ztdk41vnC%JS2)0D?pHsvtm2rpDp*hgEk7;n6WBR)y<9 z-lwZ;9r_O~e{u(4Sa~~B#Go6YipaOqHuk&RbSx<^7JP?KT4}L z5s}!z?~;* zSeTfQGQU*`I<-3ny5Jga0Hkl4bJr&+40` zKh86!oLe{k8`ev!_?o!bbWrtD-PfkRzRhEflVU?=A^<$}0Xwl9zIl`5zP`{Uz+x2u zQSXWb0_-(cZk^-^vOv83gl=pAb?6!K4o4L47p3JbBh@|7-Ed3k)mqc5!@Z6jwSPcs z=FNL3;Dd!Wm?}_?*EaH!E-EU5M8O#+L1sG0jc-fbPL^Ami2N=lDxKCO5Hc?n9A>@9S+{;i zAHHP+WeYHp61)KuOp$A#kBA`}(ZR?dloqvTrZW2n9jZs zXmY1Hi{!bxirGGg`D%F*zl%35udWn@&9kXc%C&@trJ9Ye}m zJzYStnJS2J!<&^STSwS^V$VH+w{g>bJrHpHihiY*#%=*8YeuUv%RMH~Z7llcX@b<8 z&Y?6|SMmum1M*mMOCC<$SaP=5=#4pRheqoMbd)CG@xa((IE9_0=jvc~?63(55z%~m zICb5K^mE8ZNcXAN`3UcbVWFZKB+BGsq>xssX~K54EMvvAG|CQ0E0=vOfje1jm^$2m zLnX3a>T!DsS0V&}^yMoDSC5i7oL)zl3Cl|+;;9`zeyi@eLBt$QiOS}-2T9q5j0pxq z2mVXTsTeFA)KBXcZ~yi~tM~wTCz2cpZpP>z8NqpImZw5(-cMe*vN_tKF{2st84~%z zUo3Jt(`~6pI^kDu;56(NXV-ix>DrQ3DiI^Z9xQ)-vSJMQ#$s$}r*o)TlH*_~j+ADJ z0;N=nQoS1Iv2`~>X!+|Uk1U(c@F1Fi8FahofqY91JhsBruZv&C zTV!c!%U}*QZe#A1lhWZ9Y#C$Q6WC&ibWAMR&p#m6N_4H@(64iFqi5#ioU&N(gZ^WK zj_uVIhO%9TN-R()52?TnB>W6iG#El5i=7>ZxBJEAT7#i$UEyhcCKhL#1I_>sQPIt< zrm`Y%3$iGDi&bnZqoeTxomR;GuO+=o6HyDV0bAReRAjyMB3gvDsZ;95KXmD_TKiYh z0z)dhpiE{4yDSlBL3@xva-yQ`Opb|j>7yghu=Zd$(r01gMxIIT5|*jPVU-hW2BMNZ z1KNMb!SJkL=B{Fa5)=d}5~z9>0kd7M+iCf>gp19K3TrV_gKtWrS73{^z;=fTDy@Ss zk7cjETAP_yKV}|kQOnC~~i5TNHK6_*{TG-0rJ>#(u0VM&6T5z*UuZ!D3DbS@hYMXuA$DI zHgok>xf(MXGA*IsrWSQb{*E8HxB(UX`ju}C&kb@xCg;%iZTT;LB7%Ym=o$@GD|x2jgk5SZQp0jTkOYw9(c{>d7F$n~O&^SsUXRKIc-I7K?P zOusALy~+G&1t40iQ&Z77H(AK{9`&X^QsVfou7|-Mua5E#VLPB zu_OusvPHsS9#MX{2$W33z!=KeS0))HD{kF`Bl9UFM@@=F7z?8+l~Yxld5c(EjqzoP zSRYqMFT;_e>w~*kq>s*cxsFU8#Vu@dR@e#jP(M@tqhriajoOIJ#z`dP5T2tsCP!ug zj$DrMccfD;dEP%}Zl~g=Ht}c;s9k3L=}(wanLX(p6o4q()4~nIad^|P%_qcF?tsi* zjYFvDbTN5M(PkRzv+MPTKaZdBy362cgHG)0x7q{oAN-zAyU;k^{i3Ui(mDta8~rF9 zxRWXDGq^pHrrCh+bcSn(eJoMpz{|##lz?qGLrr5))=d-?MNDXMjdr4FzqcG#w^RW; zMCARgTKL_^3%+x2dLset+JL*){O0x6)(SSf=FP5|PO;TWa&gYNhEBJx6_vxMg5l3G z0l^8lVP~8!yYiFn*;dFQWYO-Zd=|mvskO| zbyPDuY!3040a+S@Ee7&-`Qz z!V(!ox*9t#)m$oM`~&WmCPhZ36)6|EkKX3E!S|Ml?p$&a-mM3IG6hju`BDNi%J ztb=&h=@|HgJwKx@`Dx`9VvdJr0VUWVK8s8g>0) zm~*UL4t4&5imhAmuqC8b~Axssg}^O^U@VcA|n z^)F(1Snb_(N{tfe)(`GUhZokElW%{nvZQQ&ke3G{kGqehpdPV8^XbuPHG3r^IWBBc z4M=zt$I#zd&7fW&@s!{DXJdkDRUE|H+GEipvs%p(mX@1lp~1SsDJ6^u&yf?*=O3p3 zeIcRq?aouiS>NZUJZw~PT&y>9pGBWMZ5C=vi_U4Yd`)3vjOv|ou(7cVT_G-E3!lRf zVUsRok%MFON?(;&yrYJjMBnsYZ4a7RdO>;ArR6`IOP(xli_MRsJ>&+$dg2ya z&)@ROcG2={wHk`khdJjy55pyP=6J8L=sxfJeWEA2xJA{{-kq*6>THBk2>OWp`heHA zXX0~Ga&nqH5GwP``juw4DJv^$>Y6|m+?yV-YvCylC9@{S5c2AIq#b1WU8U-VUtGAp zgzCe7Dy2GxOdZox$;SSBZ>lw8V=`#dor1|yc!1I&U=6WiKiE@uo0M0Ekc`{t>RR2x zh5pRNzf?`_a8RvD6Hwj3c#5*GW>6ykWJc*_kUs9jJy3g3{h_AOE#+e3yl86Npfw5l z8>NVWV`#(_!(}MVp(eyY1)Xdds9Nj#B%hd9{TIJI^t`(o+00dmjg~Q_i?}|W4}MnA zEOmkQEIh9!SGQOXMA#4Yc?xqxNmi+X`0KdoH@8Gkj#rOv7^tsL=(j2Zpj2Yp1n8@{f3T^)4x>7<7l4xe0ziwgMZCkvags8X$uj7 zi!!Jzo;5j`zr4Srh%#h6=7)-(eT*G<2`xvs@|X0D0G13|Lna&HPcL}fF@2+=IITA! zU{eQDA!1nBhu;y7vNh1ExkA8_VU*~di{`Lq_GPpG&(rKWW_U3 zN5dwPpS5B{aNUxDU?_(qve&|w35QEf+C3f>S7yhWl5Lhaw&NDIjZ2N1x^E3T$Ns_mx%Xqf< zWSyh2ZkvCLLSQs?yc!-_36qtxw8eBnkV>n0;DLt zh)biPl_q7Q`!%-?4s`j~D%!_!upY$Na_$R46=W;Mx~5HIKf0-J6)imL-m2jAIm?mJ zfwgQ$vr)}00l<(Aze=noy72!hGm}%WwVx{A?VLZ2{hXxTQsx@!RLaWr`S>OC$ftiJ zON9QO;VSi>pb;F}fr3?rHOwV(`(@+qqgI>PV6)Hs33vc^#*57iR{;(l8~i=pIm-8D zE&Lr}nIw%;O7;xl`PfV92ld^aNv@Zdmu0L|Q_gK)UmsGi0k2F6X+#WtcwbzgI)vkn znE57D!Rgujf|!LqS(J+K`Mg5oF9gyqti#>zI>1!;+KTPRGCy2NWvP zwsaCnBWSt&A3R-1wZXKY8Q)fz(P)18Y<9>YtodZT{=EhtSs6wvJ*>L`QCg1T!e~sZ z6&~E49>Vcg^oYA5w{!3(=pcOT3eO0~S1Y@BJYoEwW0b+}*LrqwCdDrdfg6%ndaXGu z84BU5SeCc!uf+natq6jChq)4$1S|34DwLxNPxnC$PHgek4$BFO@bYSBgWB7tf?<$0 zL-CB*tS(F~nS9-OwbDzFc zBplvt2LzEGQd<=tk|?YNE;ahH>gI~PI{tLr_U!*t??2avu$}WeVE4Fv5Z`z$&Yb$n3AtOS$R9d z!_6)YyNS%UShf9wdcBLEKB(8lI`Owc0jW`kdk48|1H zEvyH^l1rx}bqbIK`WIQLoe@C1&EM8gF`Y8SB!e{raW*ql#xRyWq`gx8{v z8b>mQmZRxEMsI<&L(#_OhgXXF7*3T~*bNKsc7eq+s&XBzV4Rgv(QC z<*{S4!-;c>D>#tmE9&T&dJGML8;Wkfza7K4FGS+JV8ZukEfuI8AefNl=Y5mn!Lxs# z=f5?*Rp~zqTOPvxDV++8m@42_)Lt5^>3z9R9Rw>mz}q zKM@ORIr4Z$wqXs|_n;kMV@AM$_fMALdktzWLMcat&4+!Nma2yxp9@d8kxOKo+AYSX zwTJ0n+9tuIx+85>lo6!ZGUbLg5Jm0vS5MD%D%XysgRH)0=%JE z$ymvI0`!%)TZ4)F30jIgI{FttK0seH)_#)o6_h`sBo6~W}PM5`jZk4`%{eu`!i-4q53HZ?o^H%-)eev zj1E#OxYL;WA*V(o%hUV6HzsP&5g%6vs zI@8b4YT2uI4%3K=M+MMYxT?&^2A7UtVaFE}Nw=ZI%T{ycpM!;faEpurAv=I+sX_pvMs6YVT;v!+9DF_3Yd?tHUcCcQdipt3*Y1pyO@|L z^=*hdSubNe-ctg7ee!6>^E?rU+sK1MrwrvVup~7k66KboGCv?3yiJyQD!Rq{T#MZ|sn8 z-XgT(+Z$GHbbJ4JrQ_dJG4 zueyJ6h7rG>|MHjmWdYgmb0G8_v8UBqJsWs;6d2@BN4nanqNsjZtcp)*6oT9RKJc5g zsQ#KInwHzYC>aW1LU+e6-*-fc!HklztIC+`m{emZk2vVCrUEfoE^xX1 z=8n+w!?_`NSrkY!*2YM%SrDT^y(cScJ{t_gK_ldK(CEmqN2;aLOGE{7+)2qs-tws` zl#1Tojo~Ip3JCYX5DvT^3L)S1G#=d z-twIwf9r|PZ4ZL>0Hu|vdAC@-p2{w-3dhv^oQ2~gSChP!>c@1^GlPb+o1uJ7iQ?@2 zkR85d9GNR2y@v4p%(;<`n|zodevF|IM6ADELG~>`yc~a+4djk+27BrnSpr11DjGj- zyeG{jf`aY=Bg?aTQbjz)V=0NM41k&yPovC%6mBfeJk!CD(Mt`4F(#04(If{0ig4s& zNSo=Ved3bjdYsyWrF0l(cquJdz$V2?L0+4n{d`K<)s-5p3l}ZTrb=AGpHfzSn@>7EpTgdnN5*?$j&rvJwQ;N0B@c{qILYruchz7LP2}kO? zs`e5#MiXKtXDkAlRrm-ovcaGaGe>y~)y4{XvrlUJwZ>;Felmk++P6$?u3g`{-cx++ zAP_cLI}gO4^Xh=X>pve8Wxc;TQfCgUfi6K_gkcFsZ`)q>IZ1466qo}NL@%Y6eKHVncgBC(}f=*yqn1c?2beVmR|G6WFTU;jn_&(49YB66=( zSn4#GC6=)HFI+jC)4c;iQ;wq9pCo(m1-^FWW>j@%0rDzxSHDSB5K;^@&3)VKR@%Bs zX85`YHcHm45SfU{o-%uUt`1|zMDB-$NZ6;x(ez;uQ#ivi9q8WP4KS{>zSD^(Rc-oH znxDuu^YA};OKTBuwX3zrm5>WEdOMP*^vD{>s`aa2Wq9#`lciq4gZKKS^I&^?3#_iB z7XZv(StY=dUwWG zU)1=>j3x94_uQUJC;xnKiNQcVDQ5?&7x{Jv3~eDqt)s*-_a;Y900K%=eknOxtaXNN z*Sc*jx{M{xRap)F{%K~&mz0!LWHQv#(?fz7&TiJ*fAeRfw8zs>mUQ1?V!T+JRh;dh zAPFced;WrXPmLZB{MX9^9q5IHKfZ+O`zIvtJgXmj-mBcUy?$aMExY-xwHD}{^Q9tQ z1Yqjr7>0jn>4b6%mk{sMl9N}8W$ZI#*?BdC|A!emYvXr=ixnQSpgYkUlU-uH_v~b} z+Wx#-JOe|ovfY`vX-wBzz@qJV#0%ZYnt4hBq>K1!2WYB*t0^`0%DyuAxi*KO>f^Z! z=-;)8@7XXoMN1!FS%$z;PJMrk*WEQSS)feVvj0=c_@NVt6&@QoCFl+)?b98=>WGu| zb1aUDzX^PTf3#!sx=-cb2eXYEL1jlMvu4*Rek%>yZR8Q0-a#m;o`WN&opYx z03gUD#XbWjd-ZubSUq`kIorj1CBgiVn)K>TuG>(IvGwR0-~|iwz!k*KSVi>(|6Y>KksEv$HcuTd&+~nITMD7(DDvy7}Z6fm3`;r#P{J zc~ZM*MTZsX#w`C*Dv#aussN>2zN&~KPvM8gj~TMM`3L?9lC;L}^6IRUg6uvc z@+SmBo%LS8RJg-m-3$*a1#Ln5s|=p5b!9M>y|LhbgGYx*P{b6tP-IfBm`gAm0?3Mp z8iXf@e{)4`xZob!tm4HnesgKPV#qc?HwiNXLoH(vB`)(ZQvH)akF`3|G}R+Bmi7H= zEn1ZA(bHJgqJhWFexd8QYW?5LxF9w%fE15!0MQ~BKv;!o2Ff5>Ck7i6 z?Fi5+aZ};xH1Cn?Z+J#d&@lUn8|@S5X-2WG?91Ig;(2@hrpBJYEuuJbc-Y0~BUoZq z`iy≺_h^&eyRzyzm}DG+qdUoTr=t@@bnz6w7>KBSkFPqye2f~psp1WQ9>qpAvt1^dy@ z-U<@pqSR%2S2fq{2AqwA-4>Nl_j;)i3Q3oqlm1n&gn(6Ht>up?zyu!#>SFWZIR?{?t zfQ{xJygKI2KJCwQhzwLbb=||IEWS=bKe~71L2C-^dIS|hMr8g?28Cb(@0EaA_3|{m zri(P!hiG>Sl+!3m<^2z$@lG2<+$!Joz2k1$yC9!}JjrBuzfy@%{DSm}|FyH=RdPVU z^H*#FEwoVT&`I{(+?7Ul=$Gmh7cG=UkI6+W^+CXgE6vKjbn3aj42KeuE8C^t{*|F? zwPoiY_9h$q#0CPM9o4jJUo*JdCHxxo`&u<8^cR2nix@s8pH6@(&+%Md0y6Mq8YjPk zG_cB)kMe$$5FCy!5!uA}SvJki38!2DDIbgeFXYE_^~!&fHL1{{UKo6V(b#*MzI*w} z<})R_vCAK`_dnyECJJiHhpnw(cYMR((nG0+(TuttrX$@#IpEP4sq`Ljs)OJueplOD zCDwx+c5s*SJ}gR_$n0K(8$ifRaLfWhPi1JE<#9;}@{{K!gw!>PZC@@v@3n+0JP(n? zJzQrd0%TvI1l0Ojr?DIn*GEj~08cy9DvZL?Qk;7EAhHPlEc+36dQVeAaD~pzf_6aN z+{V`rfn$CjO?sTtJ{b3Xw3|hKTM=rdC*AY5E);m@3N!N97Jh#O7*8Nbn7P#EIAoO&KK!KvO@;%Cthyg*muBO6oEsqQ0=y&Ay7oVelQ- z4z%Ggk*frlADqqS*CrajZZt~5Vft*dJTd9%%O-JQVqIMweMAPu7=$}1aX02Gw;rdi zQF%O8;zdwZHl!$LOW88oqzUkSR^JdEv-`Dq9hw8A*+4T$_G-v_kOX){I{Qhr1p~>x zZu7U&=w8;+YhCKD^$tT0y?jNZPdVnh>EBDD zVPxB4`TsEWmH|<3?e{k^z|aFocZyO&O9%rfNQj7Zcb9aCFtkc2-GY=zcQ=ZJfOHE8 z(w&meo^#Iqd;afyh4ZbwuWMcFv-S^~hLZ{Xmfe_Y`bJg)5gm8)nK$Uo*@m{hb5voH zi9o$NhJep#;$fMh^vKK#1MS7Pga3@4rgB-I@?{&uLfN}EGm7#)<*PG@T{7DyO!ehO zD~635ek@+7taZtt`M&;PdcNF($_9cQjN^4|s$W@^AB7u`m+LdQ^=hL)D^Ecu5Gy`wKUlc}FbF4CXA8tMhd&(0uL=V=2ku+$U|co1K!TPoqCc*1{1)K;u&hWb2eY{de_)LOebs2WRZtJ3xIzJ8vZQ31MC z%9C^KNmK)Zf^d;)W(=e@vKVxC>}97U$^V;~|D&@#B8d>ZGqvSO zPb9QmvP?f(MabTTV3m@~Jl-8v#(H;}n6$B^fHpytCgP}0o-@LHRZ49C4j~KF~_x_igMEYpmU3p)MIA^isi&P&Bxbs>-xA za>LDFMI%|*BOcM87;YYk?+InGGRga%uL7?X?d%{;p`8-((-=r%ccxTwK%nK+iTRXvwg{_ z_=b{e#`86?ak|a6W01jF{GJ`tb5`u_G2K>QhvIr*=mdR>3U3@SynE*L()qXu?87v} zSWIVIg8~BqnP8R{qeYmhQ>=tD)bb%=BwG$=+<9z#yzs4Hshovjo6cvRc%pyz?!Xar zX(bENMM{K)k?4dt9`2%?X;&#Biknle<4HKUeiqTGRjK&@K9&l3lb;h`4@a)FuS96~ zx*iJpC09^4R_`pIQCUSuVI)(%kBegkv-THJ;QhKN<=6lB{eoSi_N5#90Sb8-@@rTn zdgX(UO5xB(hvNJH&kGzVll8e+XQsf?xBUB$Uh=NO3_5`ux1_OL4OHiJhtMYj`-zg- zTR#e?9wzi|k71>7K+l17KM_dTRBRm_|8YGC3)IFgP)2J{ZwXn{!@cF1uT`mWh$B%c zt3C{@R#nY;m&e&O(VCibLloJD1+}_ZLOE5yHga*{s|nw$8Q!-nWXr;(1BQM(d;4(? zY=XZIqZXgcb-^N0gg?odq{!t}ocJ~s&u3*joQ;2B{NzdM$&){}|H}f%)>+S-ep-9S zS(9*YrnE?1fC=U<%GzT2u8Y6qQ?SxeHOAfts^oAsHnvC25_n)ClknP&2Mx~W(*s6Q z8z&4Z}frtGgK!W=g02qal{@{>uVgaxw=J??*L>_5!Z%%VQ<<=j9sTPUvab zc`O7m*t2kC4>0lhzog!;QH(G@#iy}{VQ(>E<_1UYP#qP5f1Jj?o~u#s8?%e ztf3WbJ<3m{uj3+^m)xk$zJ*|0a+sZvmh6b*bKuW%yC=>GyA>aCqgVsiQd=kY8K^gg zM0CV;usU)J-OEXreBDUFTQnrv&HZU4TdWK+M85+l+uhW}$Kr^vvA-sU6WqQZy31$zxQ%5Ms?M)vKM9i6zZ$jhVt z|2G#V1*yH`_<>maRWYhGJ&*fux$^Z`U*+REiv!SavP>rE*nsEbK3@XS>kt1HbYNW# zAqT^m&P~^HfeIxP`p?4+0f9DW{2UD1}VZa4$1=a zjd}u13Bvf+X@6ko#KV2?7>zfvdc+j8gZUD)1%1|H|MwfSS;0!MK2>Q!3QZ1dN(h#M zZD@$w&=`CZM~AS4K2v}O*M9wD+@*j;YhhMkL~7k_nuVp~a6?!=4*tX-31=bqbEVLo zOV95^F;cTRi(oK{$L&Nyr>6|)rH#1p%e#-|GA#lMruTc94(CwZ9Fmo-X@)0RCwLR8R*&#;vMZFf*obuae%V=pl%pBiHk-(;) zj?x{*SE>fZl^I^2FX`C59;owprr9v?tLCWyX`F?-)@1UU3>xYRVch6CKb`K)*m2Tm z-M5tMXx*_QA;V7R+)y8#U{Ik<>;N zb#HHPY-;L#)bRUvjI`0WNNQFfm7MsfaBrszT1Q`<*0p)q-hTiP3u_E3WFN}y2N)F3 z-UT3$q=-6arqqIBOALt0KXyO@RMzpe_&PU15)2!T?8Sm zh|qa@D;^j_^jKb%{hqoV>uVIC>;ouvs;YhP%^VMPhApli$cXo%fgR_Vom3~qf zJh+SP)1rHjwbue1`U79){q|a1zfoH!&mxPt4I4Ec>U)?>7+-H{qoxHASI(nHoXEpX zSKS)78g+rn|TLbjnmWwA?g$nW^7&V8RA&ZMrZYwKFB-*MxAd8#NQgB(Z|Rwf)X z@IM*1Q^W39Gt=CjLXH=Fe0?8KIeP6n3$Koz#@H_o)&$(%jFjk$>Na_n+^6gj(%1;> zfOGPr%yyb(7X~vmlWXYcyVXPxY;R~)Xarg@UE|9)61%&*VTOf&?oph5xjb6_N@*OwFlzU^RsjY3zV63UR z%GdP!^62nzKmZI3$Ton{y1~(q8aTbuUmf;t`h*{YKW8i-MZ-Qljclkx?KZPcO;7v! z`!y>b@ZE55{!}oTUBJk*2z=a?xLksf*D5`2q9X|I8o)agqA%qX(eRFeOan|)S_K_~ zD0KV3Re~C#CoeCot4tbUgt4$JwoYO0spnz%6u}sIk!)jN7RGzkxXEZGRbxB)EwAwW z^K0;Wj&lbP^gaG$5>|90W_SybA&Etn?C*bXF88kgC>R12H-gbd=d;5?^Yq?JPwL{6 zU`uT5$>JcSdhT!QnyK~ylZ;)*=`WgG_Rl|ABmMeg+`X{gDWlKXsAJ6Z$$Y#sLFDo6 zA;Wh>s?<0C!aYH5R>PSZd+XDvb=IHKGrIn!^3qndtWrwV_}Eh)fhp{>{hz(pXA($Z z>=iK9be%Nx!g|Gs5*aIPM>#wkWMzXS55KbpS$@Xy7rJR7?Y zdHS-l;d?pkDBEQnY?3Ncn|B)iuF$TI@KcID#qvpfPtty*^2`3`zQbr%@dQ!C2CQ)J zyz!xdJPxwH{((oE=|5mHJ<`U%HZ)qtJAtn01+r^Pyj+A|DXTnJ%`;*5aQXtJd)W2d zr=D>9JoM^k>rr7rCTTtHOO%7D4}2;GudcUJHe70u)2A+VcNBwg>^O}KmM$rjLV#66 z9&2$6Y&*u>w*a*pki75~>~OT4skS{_&xyga!k2T|ekwY@lGuO7VMVz$J@sA8vhNP) zTVIMngY(WCls`)O4=@GqULK2Z^P|S~E%(5jyt;j!fc*+rr-aVkF{9kG9br=q#g17r z8#G+}vJDi{JI5cv=5KcfJXD1SB^kb(g}c)^PM07SMY}Y_B#N8|MdK4o?gzGiAr9KY z4K309Yd@*Q8Di|cgylv#Cv*bF#&blgcz0-@X3s$_X*(+%K<{Rgu=KpB?5m5{As-1@4a50dv6csDU>_YVyP&SDW= zxb@F$?7S5vf$q{y3CQG&Ph>ZjpVr<NL=9_G1<3&d9vGzD<_C!VtyljUwn09&`exD}pT{XAv`)t&szGOnW ztW-}{t|y*V>RxeYS3C0LKc6kK3t|J3iXf!W_kb2=&5zTp(d<)j3JjmIn$>Q7CZ&HA zx+8Vg{q?VMuG{S3$Pak1bEHMSl~VY`PwFS`%iV8+&N0Mm(XcC)d@+zQ!Q?xPbi1!U zb)1RC6D4mDP~s<(_>F67`w(j2X@SrRwL}t~aN?IceVDTg5G9OrmhkO`7X8-i)7dO; z+~U#%-_vbd`|;(SIh7bA3{Neh#u`-}iS^1){$jqzHewg+IjDKbm%?&i@O^6sTfLRN zrwb643;DYj3dhP+cX|j6i)O388IVEVPiQ^gYrQPRi%(BWn@wi9X24e_y@DU{-=VPI zB}Xhe_|uc*pgSd*^rA)Y9)ejSz z{C?W0Ln0`LiWTkr%@!bC!qzpFHa1B!gXYp?Pncq%^<4Cx+f&S4;lwey9Ob3|TP@>I z4McDMpuA`zLe?Otn9Jaj$KY!k?gj3JlvHk1s(OUn-$+z(9XHrA_fg!GZ^@|wPA`ZzrfkgFJCuISe zrSHmLqwu2;T(WYS_Xy@d-7i$Hx(Kv5)tEek70}7o6LDqam+UsT%G$IrpD8DDKX-iD zZ{?&%j;6*Dm?MoWV&S}Na!4dYjcnu#3NRwbwCxbvnUS&{q+3;j{N1mx&Dbc4Srur6 zdRMs)mxR=GeLu}$AFRLKxv_GWYRVfQ@+t3`Nub=dUDvK2LH8h(-w4#rb8bEe+mxFq z(Z3s!gw}y3av^>k(?jQ`qBdy)w97u81prG~oz}TzBVAEGocQnMuP?e7Z?XnM1K{Nh z5fYfy9>C3)FA*($`*z)FUwcySL0Z$|Xjs~3s9hIT}oIX1_k(wEaiE6qoPWc3AA zal-Z!-;2sxH!HRPCtKz36h$YdjzZ-GrSEJK1M7?4ZLb-+KWC=tg96RhR|g6*YK}ZE z{*z#^)-uhe%=xAIgf^bO!lh044mUR?Y?xXJ`|7G;@ezYWUKjsE1(9kzN!}pbEaQ<< z-t4PL3eg}J>V(Wp-n_iL%}vAxf*JXTBXEr-)7d-!`G;R~+QDlvRv=F)a=SSvGoa&Z!_q8xa5r50qt|#A zBmOP8>;m!kt;xVSGN~zxpgXdim^Ui()uCdI`;hhgx8kcb=T=>*+bcKwO1*Gb%-!kX zyK}%`C;&#ON0|y?0Q4RIP1OW382xqiN7NOlzhCB2Rwg`39lr<6YE$ioB=KXO7^_?t zvR5!=?A@5T5?Q!S{-vNjb>nSKzA1e`&aOY}ohAS}l&Xs`5#JF~gKt^i$u!ogQ$m%_ z%DfE_-87ATuaHK>zCpG(DWZMtS=1YDkQAhpV4A)7%*228Zc&{n)@-k_)@}?lhIf8; zOehm?qF)4@P+Sd4<*fs2T;*S%UZuyiuMWvOyt}1mnEF!wa z7o(>O6zDxPn8L9#aFk%zvXOy5T&&IR0)K2p!(R2MD*k5h_!mgK2^)s*CD|*cHlVgz zi9ou3^z{kUHIOzzu5`t01R{5`6wi(9xY#n3&v1Na{onYVC$(MWx&Fu}C-M%9q!Cua z;t7lWDQ^B6a9;xc7w)}7LtcD+bKev&=NNR!iKy<#EuO#QZG3W%m!C80XddvFN-}l~ zbSroUO7x%y`18D+$wl^{6;xE3&x$(Hxu1!x-uVlh7o{_1miT!mHIqj6gU6HPoq`@`S#6T? zEL4t`)#mtRi$cF(Ww_wmYMZ6+o{#v@=ZCUGy~Ba+!}(Ywk>`K= z9VavJUvoCRcaT3Kd7`Y8U>w<|_owt)eL+`Jc~chV(#TvQ0VonN^C_V>&OYZNF`l8A zR;}i6LwqUKVn$BzE3dBi0-PIZ(LZjLHmNq?%9vR2P&Fe%(dK*+g-Ghseaf4%fb0Ip z{R=q?#H)wapj+Kk8<;o|<)Xwi`Cfn(e45>;O68IYY<$hYGi66XXJ&*bT;$cl3e=S|kO* z_Sw#?w(|_{?Rk%MpSH2T<*poeGO#<=b7DQ;J^K|b3KN%_e)RR4tYOGkEn zy-vcTh6%q4UTPaZFYAQFUh0!iXn=aYE*RL-sdVOAIkc<2&QFE1zOt*mi9#pbw0)uFV;C(b!M0qM zw@oBdDJD0}qytOF2Yko&`wjt|PeD9YeqX1_Bp`~cikJe)yFhozZG+EE%w*zzGOqnD zLqOirrdzsvGY|u(j|O^2exl=n0jti&^VDdx1?AcuuJ@YKU#+IptH&eJML?3thn>|{a$IUNh5t>T+*>*u?f9?NwtOqUh<;-Y52bqokjg8R~!e0 z=N=kLUa%;FxQ9!w(|%FfKgx=%GvwAfijg}K^5S(HV=Im?rJ1dU^%C&Tv%)dZ`Y?4K z=#c9}(+TRBS|Lgh9yPToK#{!W=68vf@p}jhr~b05gcE-N@Yc^vMaj7D+A~|Rf0khG zQbFZf__;(h1ibf6R%X@HJ34#hz0&mVrePlzIaaY}l@o}$^56I^LO*r4|8#dG?T>)4l*~oG zHC{2@!hGW(RhC=AY45$*#P{o|HV)+4nth3wm#a}fxGdeZw5K(7IV7S!N>f{}{CM&} zK*DhC)~tu(USW-wbpDY^O4?5&*V*bi4&hmG?92H}g*js=O^n}mKVO^B7WOJbam_1S z?C&)rH0Y5#cZeo_^HYV$>;XwbkDZkl*)+1$wAU(VSl4kME%>1If~vk%&cx#3cjt%3 z@Ga3fmtY_%-2L(pFuaXJ;^87bKVnVvG0GaWDVH9Tx{uS9J2+>bc*diFn?{*n> z2|Yfh91=H6+5el`AaKt)C|$(+iwdI$a^=r)((ToL<07lj*e48qVzH;dGptCsJfMGD z)7-L7PbqjEHi3?payfL~lFMEoTn$x-Zu*mIk8vLdDX=usDc(PNjTopOmr1DL8*Wc6 ze^h>-eo?Ad+$M~Q6v&8`8{Z(9y8vkT9PH6fAAFTfF-E@jvnt#)3Yk&|tN$>fghQap zd~PcTDV56d9=2Sfqgt#^(~L416<=f?#L6RLzkCqOoBJBwNYL$Qch8M z&)HpzQf^7jl#^JiFz*s~beYUCfe6R0DDV4&-5nnLTm?OS;WR3xD801yo(Tbk>Mffh zcb$fS7%N z)Jz6eC;E38nX;;PiFkX$~~kfX%$dnhE{v3>7^2&W#h&e z4+-jbU!&X()8HnS#-Sc|1{&}t2M$brl~7dJT(hHa~H3blLcD#{KXT4#3{DTX*->Ojut&2 zv`BjCsVDLomzoK#v)RL~GOX$V%s+=GGrXs3Uc=3s>-0PEPZ*!~bAr5L~Gz{%AYT z{A&i&dz%4`P>;?YuC)ix)9;(xA87sU!inko+mn7*;f#v8Ez`T@Rr!_NUyId(d}-qX z)v&`MMxWC5meH_z+SR*&}B$y#Z>oMct zo{O@<3gjiOidrJ-PHO-90*>kCvINmqg*r4^_EW6B#=^8gl~uNjTXPGirWAr0iGSAC zinuvxU-B1YRCyOE7k|qf)f1UyjX~T-b|FL7cITFtwC*KO3~&u3eb|vu9-45p_>F9N zT|yspPa^$@hzgEa6j8*#P?-Cr^_8<21}mR74ha8UjFPE!MEy&O8sct^bu8r2|8Co=UCV^>SR z+!F*Icw9+xzyIIZyPX>4X6X8m_Xv)(4bx|VAWKUn1ay`)+(0bWdyOswhP8F)ZvFGi z<^ZUa5iJ+hqh!SP z@N&EAJ}%}i*k%bJde+x^C9KD2B#AE1%`{cl-E@)!tp^E|HUiC;C?EiJ+YCoX(A$tp zxwSNesN+EPi8rG~I-+nN>A%vMFVqS4Hpe~$F<6+KDY~h_2vsHfhJ(+~Wj>1c)>Cd< z*jUvb|9BvPh9j!d8^%T7?P_CsCHmR0cHT60@#~OJ(wMspeT~NWBy|OSBh{+KUk&kG zYNY-_-IqNv|EsV5*vf`zyy|+?J>l=>+((}>ynT2a112SLKy;nl{*8jSHoy?Wm2ahF zcjAV`Cw-H##)G<#K*&?s)m5NUf(5D?MMdJWQENTKuv)8#Q%tl1(c)Fh5(;@YFUb1d zO~IF;>5@Br7Q!_}7`EEk_b~{eJ*LqoC*AZbr|i4x5QR>fo&7M#tVoDOm&v-cI3aPxS2U$$!H*1?aw%L?_G0#JLEaEfDw! zagh_fC0Mti>KB%02_WPPci@1Qt7Ya)9SKfH*wAn0CN`kKYo8ImMK-{gYPOVISSOGQpezEj(^Co(i%IHI}66g-sR8^S0VG^~8hEQ7J& z-3VHe2*kbZ_(h+<#j$5ne{=62ItN^)rGpY5{p{g{_}Ti;uClcIO}_diC)=kA_B5II zR3)`#dkbHh2idf$S#02SeL&pP;lbXaU+d@Vfgi* zimQ*dtPaNieqz~~1tYkTKAkdm-ECO=68Jqi^y2Dh2p{9&?3)vI5=?WKnx$M5gdGSG z3u@1S>2M)PFkUjcgg-`;BHgh7lR<%(%a582t!U<`uuzX8NfpRt0b|9+^g1-FIwib8 zJd>%6n5izmb9uzGEQXbE24G>v9z1L5ZqY`o0`j=+eWpH`=3@1%qZRb>)*c?e$m3?~ zfBls1#K6Zx3>@EX8Q?98Vth$@KE|FBx0Y6O#gW zs!1;gs=8QBR^&QM;F)E8|EQBXwTgISDWCqo;Q=szFmy7Gn_?k_N@Gyjg!xEW4T&)W zi+(QvTkgJnn<`+!V-kfQh|Du=@|>X!UX(P z@0yQ)aJgRUx5f4Rkt{Xy{R{9nvH2!Bz73Ol)lZ;n5|T!@H^AY1Us8IR+U8k|{y!H= zU;=1|M>N-*)oVq62iRO6-tVs zJd2Cbp}(hJ6g$itkY)S3Cq}gDHOL4;zZdd;$W|tjz624t8Y)V(yOseGno=O_U$hv~ z-Vl}O%B9!M4YLyB^;tb9`x{r){uK~zFP zD-eRg3{ySRXF4Gii7_+?H|aNdP3jVasQzWI3OA!>TZEu5*sH2Jvp@_VFT>o-JzJGm*p^DZ5sznVlcms&54~KqQ8W$ zt+ylzbpi8L4wsvw|0gczFW^S}Ep3OTj+McWYw_XEBok+NpJG#mk)*V-vGFG}01W>A z_?l}MqgN>_E7R&5Owq?!GZtIzsu;|4je1*#oYdU4lD-4bnRI+C8jJlH$1CdJ?p)& zPrMP6`#_RYCN~6j@EEC!&ptE+%N{#QZI>MHsSI#K6N0~i_m*l_hK!^Ecoj5NFzYqC zsx|=3^g;UOAVxvps@k)cJt@Ky9|sHFY=r3)AA9>+QCBym9oU6xbU{iJF9z4!P<+ZS zD^0tN$|x`w1Rj=Kz1VVz^6DZK^?v28VW{veR&{&1ZigOlIg@~YhZ0;XXuOMoEveYx{C zv_if)R13G47l(s(*1yuXA7uH*ri~h7)1YS^*k{bmurrDR+h%7R8h6O=HaKQx#D+`t4`XxpQ0O$gDWJVu3_W$TIs>uNX+LMEUi~Gy_T3yxJ}^W^RY& zh#Trz&wSr^Nqly9q2heUitdH6egz9GFKr$4PulS`Q2zb6@g@s3Tm*p%h3+oFpGxQ# zynhqc;8f`>X+3Hh9i9f6aQWEpjb0r*PJm2GQ*kz(FTYD554l{Vv9kL#dIL5{EK6gM z1Nj;>vl`(6?mGI5dF#MbFj=5Yfj09sE`=(TD}n@xF5FCjs2;)EnxLYa%I{`CssRN& zw%n6yy0xmC`nsG?TUk7ynvZ`mDsUI2p=!Lyz7wi1PH+RDvS8zz!=yuy5&Ol0(}u5c zon_p*(+piEy95J^&w$AP;49^qfRNSrrV;eNjIYm4I|8MPGvIL*-m#8PJ1F zrJi3@cgD&m&cWP1K7q2hWjmhe9Nu|^71TcOA{I`}tFyY%rqB6hw$9oB9p3pU+M{WL zQA>zwPus8mjAi=q;-G63|FpS|=n(NFNTEEq-8RHYzw`L?wL`Rng(+@-Y#iiqWdsZ9 z9rTBoZ*jX)ZZ}hQo0;xsE)BX^FGsFZ4@IJruwt`4A$^}tp<{xXW^GM&dtq0YUr=y& zcVIi@?3{4k(9m#cYe&PlDT}P$UWkN&c zRG;s20c#Kc8A&Z@16uV;r6CPMRl$StUwR(S{7d&1GMJ7qY`3g+tIf=1%P)UJ%Nm{kx1u4<@$tHik#>>bi z8GbkAkDefd$#F9Nyy?zxDH$W)G)F+0Wh2AV;Bx~Ee%t(M)VWY%RIXg79IQ%% zU)lKCgREBr3tm0#ytIT>Z~3Rl?_rlffA=t1iPBmtE8sEA4Ov01c2=`oJV8{=;p0A1 zr}}+QwJ`L6f!F*rR#U9ob1$Afv^+vbgK((xT-l@)ymoTb@tD{&M^f0Eww(-frS_&Z3>_}X7l{Aeo^=UL+gXqg zXY3c4fcSz%1eG0{DRP(PFrdi?_lK?vW6M#$Uv!kaaD zh}4{-JktnlcuRayzki!vdIY|7BY+W!#tO58e(rttWgZnUr6+7Cn}m4wz(fER-2u7R z9MJWf2GSLOuZLsRES8)@Jt1+%@9P((;b@bpY`JiyPY{bYRmQJhxAx};tM!88fc`FG z1yz(HX#B(>q>iOr%dqw=Y^nDocN+ znAwiJ?18+aA1AZhNd6kKPuUl_^g*LBhn<1fw*{?!ASD6m7J}{)Ar)++Q*HCXVso|XDqDxblvgk#Q%Dnkl?KJ61 z4`>9G_GvnBh+L3T5ToTK)9~x$LaZPk;A9YTn1U;ECnw~Mz?ASHFiAGxJK-3@m5Pt= zXzHf#I#QWul;fd`tLaeMzYx+p^NhZ2HGdS5n<@5C*fA`OUMZ4^h7FIx=z6~)S66%M6d+*itp!!EoB(A zw29kw(KQk-c$RQE0^{Dk1q=aQW)!ps5&be zjHlUs?CTg`U0IMVt+qHb@w4BFoLrUY(=9giW7H-_xi51R|cXW#r=7mVth z4BuVuac@VI&GxVIqrTHDm>Tov3c{TS*4X}7DjMq8F}AErS*NCNK2tikoFvNInb60n z{~(AX$^L0|hk4Tec`7}nk}XvX#)9`*SpL1{K1_0tAjNRC3+1KSusaw`1T-;1BW0J{ zSWXvfr#Z^Eog#yO9Yl~pHj-{3mogUrZ6r%nLu|QEO zbg-P;JceLwtY$S*M=ub4ab9ZdE@tzFaUS6?7x(?KT!ekD)FP>-5P_oGaNBR%Iowy6 z`c7n9d+_RjSEzOG-q<4rw&_4G7Sz5FVE|e5z<1_zTJ3~$3LAXVd?FIU<+}E}vzS}e-Vn0?GrE_gjp9lv?Op9i zYo&jXxC@;GrA5dGcLXuVv2=-tL}!g4xIt>nf$-wD?(J2bKTTq#X`(vBB3zemC>*ed z92K&m^fluSu)*g>s@9NBYV?rV?vNzsw(CT#G9#W|H1mMbkr8O{^uE$#8x`#^HG5P( z$TX~P`>>xcsnPhEN}B-pb2v|Y%9zACq+*S(p+Ld zBKGaFM55|A$|v35&n61@H+jea)&`o~6KxJ~MvaBrBnuAuIuECP(oD2fOhe?MK$G`L z?AtUpr`n!1!t7FAL))8YWh2p_=qt~K{U+cWS%E|c>}}`a=eR3>njU4zzAE;5s_2$7 z#7IpvuJxiP*NRy1nRMUv&&(&>WO_tYoI$7U$^7OP&FUV!Sv(aWXxbNq^g15XWo0Zn z{LyAxrrP#j&}@G98kU(tdv=>2`q*{+PxFA}56m%5qlF>xJWrvEMsKGJ%Uu`^WQuoC z_OaF=mq3Q-=UXzn$6WzbSF@Ft@3aKm{j$975S)S%=YpHa8-9E@c{VWl&|xVQjiZ?} zsQvj+*3uzp&HEXlkoS1pFp5)7f z7WaiH%DcPSrobs2Hv5Ymszsp-sRG`P4PPoU~NLX+bo>BCb{`6E1L7$Y8E>ovc|AykGF+%EBd4y-ec7A25 zfReHb{$)(0%MI&`HPQ6T@BZeH(rvd_YJu&O`db;;15=&EVZ_k-qOPw7eCx$}IK<&7 zLyY60z$4fL4c$l5dpdurYu;46856CG$sfv+RTY02;RWgx5J<;jsrK~=oM|P3|HnNw zquse`TQbyyVCHACVq`g8hpe~_;cxObR@lAC)L%$~PP~05agfhK1A3xkrgq_Im~zBr zn?)}v0BU1PGWoDCMn^~IUq1s@A3j6i*>^3c{KX9H4smSE6Z{A#f`cOvuQ3fHnj$X4 zw4q*123@xDJVr=AI(iV&$BRG@YgkPUf)$OV6tHULdP>t=fMOsjOPD;Do+9B-P^o(o znh$5f=0>n64PXa?agkGyF3(`^Hl)?~(q@;t>t2QD z(;!dsg%^?8@#6WbBY~>dSU=%`jh1U8;y1G9)^4nGpvz_>Rd*LouIvg$;Q8* zr*7&!Eb}z@2(@JGGYy0CHArDMS20cn$CFU`)9ZN&|AZD+}-Vr!B+9_>{+vUHOO_$`I1;)`J=ZP-EkT zU_>kcG3fg#we?n)93~rXv$_d*H@v9r`Z6vsQ*eGEtis?LK!i+M8CATSVUhUNlxEO5 zhT&oLa0tj*vKZ-L)@u$Xfuci*Ch78}Ghg1dEU@VBAisV(NBKj0^<-c}F(B2IlIK^$ z9OYM6YH+Vm4w`e^8AadIdePc%~tgcNYl{`2Q~ z9;0DeQd3jY*wRBsmO#X3EGF{H4v79M+i1_78BzwjFO&!3)U+d#L>M+mCCgySAuh=X ze}i4M2P9vEq#hPbNlpZZN%_3Z1-yS)*T&8+Ru|40V?JP2&zj5d9ST)1`3 zh01#8)h+nGlR96-qycd`Y%YP6%|FL{WN3eiE@JXUxxl?SU)}s&JH#P`!DKSM4NVUz zZ?H`+;K-Zqu!rwa4=CC`)$Q>%L8lHqe$NulBo9j^s3$BJhr_0TFTJ`8UTUD{%c!m1~q zvS-wvhCk+1&@q1jgv^U?zB9fqv?qI0L#3?RQ2L@;mhImkzHx{NzVBwU}4@u$^YO4VW z7CH@v5O-NBF>i(Ttd8K0fL|=0q@6LgajeYvQq`90~vf%053EO+|uUSkdPXF zCSt+{HJTS44Iv1z3tU<3S*;>XyLqHTAaAAsY>9~(h+S><^F3EruRQAY*%A^Pu((vj ztI;H-;z{6gi21RrzsvT;lpffXQs&pqI}#8V2Gdtk{^9qMx}gtb5r)pur*7h`Fk7e) zhzvgE>{a8Eqyhck!2VQuK+e65_X)9Jwq36l(l~vP{tbLHtukms)lV8H5aLV_Tmno! znC)X5DPvlWdnQ?GxXCQl-$CZ6wXrr?^{S~Ewv{!?Kv-wXYDHuYAETF}B8KkXCwNKF zd6%ZkqTz*B`ASDk{_Bnr4kv|0*!r)wHZ5wi-%aNGe?W&3*k4>16uMbnSkr!&vN1GiHw=*G zPkg!9j=u3B=&J2RI1tka zLGhEqDwpzJtq(I#4}qGTdSP%CNhE-vluLOosh^j|lRb{M#|d@qTEs&0CmC%Is9+MMJ>D#R*Z8_!L-jXO8T|xh6DbCc-~HS_K5eQR+;sX-wL+&ht}dqjcJf8~9M3=6ORxt~ASbJxLIUq_wpg@YE-&W}e!8`IU^B7yx$HRXeW=7yE>iif;-pwMs{SEt%X z$|=s#x@UPll^V=WFC)1=$tS$?qjq1qvxF8zy+FhSKM`?RM074bl;uT9WwF~KUfAvr z_VIDLd@!d!Rf{3Mw9^jJBb`|nRK!ENj3H;pf8V43UMJAlu|xK=p*vX0EyGry?TJ$2 z(BsC-BtepZyGhMiLl!AA4*Z zjY^XKm2JHjOJrXr?q4^!_>?D6 zPmOjcFP_8Y+ZmyY-^v&)Za6>oTcF;Te-+Ms1+;92>|Qhy8*bXKwF@*fq34#>JJx6m z((1pz5=@D(DPns|?Mu925H_*~5`c*u+5(Y&t;2^OK?T8lgv?w(+Q}z>1CxPzkoJY4 z7x1-Rso!K32C6+TwX)v2e3vptwH<6zBG4TX02~+_sDot`T^oFktYjlyTbOHo;eKz| zXZyB3grd1qABS`m>sHyEZcmSah_+F$?9NWv7)2Z}lJW5NpJcGo=-sqSs3zcTgi5l;Ku~oNT0jp~X=yO;&A| z{sr5^)!wgy|H}e6>@*jH?ftDarroWMJ^Hu2fp$OMtNZ5G@&A8+!*Mtz{U9N z5Q%`Zz(Fo@i7vp0M6P$714e6qXhj-fTKP0T4aQ0O|CoBquqeB>4U~Za2C1PNh7yqO z?rxM$X^@bH0cnO5L68z9r5kDK5D<|TB_u`ZP$Wh78lU&ud;ju>hX*oq-)mjhnL-qL z8lKXYZ^rQJ37rs}e7(PYdq;kn{@TfIr{W}qFz+unGDruyF07S0eKRk0l6Zr8fCVSN zMTN@4rxHH9&-8lCoaqNr`w9m$^C3Vqm#>1KiR(9pCexHPw^2VJ#BL|2~d6RgwViZQa=WB!3y+5ah?1 zcWkCem$=qmu?|L3jX`#>fN1wxLtZ1{`4W0q<^+X|3ODYQ3BhDbA+#a~rD_ zV;lW*VkS8&X(-mWLxOQ-5p>baqK`8-^}Z<%iG6}&Tini45fe0Mo{~K`Steg+?j!L; zr?kLP4a4W4p*7k4JH7H}+$^e@wH#GOxeLA^1Hj_-w58J0`X*>c zvqY`w;fiABgx%7Ob5jIPj!~%a2+3!lmc}yAakoEb{AW!pM*!XvTB&{0VZ=7 z2-z%a`&Xg&CF9)i2aRST{f=R87Op^$dCONo zjo;HgdTwFISYz8@`B$$UxPr>nH&?KK7}1>CpBB&D`Ol{IARb0D-1X;koQo>R z;yx1wG12phy_2A0!c3ap?%=q_*9jKX7Iq!TODFRc(@sYUis@7rL*h*q_4)(1;vg^7rW8W?+0P#9BAuMA#qKQ&{lll0hy z;0d~-v`H5yGvH+UG@3gz&mfG1H1B9;1|bo9wGR;I$BL=<-P=kkg_ORvZC(8$g4EiHEwm2y zT?-Ce3I4vILmqQ(1U9z8JJTj8e?uI_V=(hn=+fp@Cn#0-hRLhYT5yLgW`ZvD~m0^Vs!#H0T z`F;vA!oZwyE}T%i=@}0%B%VBQ%}<18Ww#ek1htA z1uk=P2KKH%z-2Gh`TRF7=>>=d426>>#F=x=x|f~rSq?%zbx;&hxge*D7ALH@%| zVV3&d10afkrn%}9<`V+x6wA2WJRQzg!R_xKI;OG*_mU$!VG6TffhXNxT-q*90>@i$ zC3^3sAD1a(gk^~KPq@ouzAj7?b%S6fnZd!_MTmF$?`&kvM=w1QbW}-T-!A$6v*2&W zBS81^)$1dP6hAX!owTE)Z)GK4FsYOmOOo zxR3=hA+#o~)K)|W9~gAhKt~BpH&`3Ruzrq~UpaZ;w<^L4b-IiGhFl*-{MEi6_KQMv z&d9(hc}o4Uc18q8s?3iBxfdgQ7esxle2W5rWulezEs@Cv%8b{c3E_VMFiBbuns1(; zE|MQ7^k4doL^cT_yP$3_(=>5orTA&0Ow{^(U5P(tGGx?B%zQ>D@zfS2|9NH6VQeM) zAHIQ}wl8R-)Qb8e8#EC#4IaMA8FeO4Nu5iuWEejGJM9yUpr`pneHd8L`NLJi1cnJr z-0vqQvaWF5q?faD)Y3y=ISA_GL4>Rfk%DVgzs-1V1Sr73` z%9ao7o_eU5k1l)E9uC+U6)+%FIAKV?I#on8Xuxe!0d(rS zGq0$ag~tq=^JoNayYN=*h0FDCC}QAIt$d144NJ^u;D_tYzYs6Ma&Bjo3pL}$DSoM@ zkKtKv_f`TH`;;lCrAnNj&jH6>fF|KN*!J9Ye~7h8JBPpcP&Yj8Ve3lJTVhPT!A<@2 zEkXq6!5}ZxG@*dDK9@MnZrm|G2=XT|GsCcM%ieAV_8koe%AD zT-}J7@ghH8&*h~Lj-m=<(?-!j;rPc@wk{kTLRvQ5apt7jmDyu)Yh&Zmr0J{+lL3VI z?$6k$&k8whASuF}689(_IJFm>T?vLATIG)w{*>p3XoVmVp=rWRGf1EEx)y;!zD+hv9?KMj$jfl?QW@75N_Msr+H&GbC2Tf7T^t>wR}6Gh=9 zk`MCShx2H`7MA+`WNqMyzf3qOV=@vZYunuGI&N=k;1XQz=&qOR`?V0g9$w(NKdv*3 zL4`9n@F9{a{5Ri1hzli(^^~ehlwfX;kWml8h-iHbmxV3G^&e(rkkcIEtk1&-U~!5e zVP8^Y@CBX{&0u`1+wL5%xhDQZe8`1BfZ2reU2JkP!R~zRi?fu3Ugev@mT4><)0eGD zX9{MF%sW{;<}8C8{d*7;?bU3nr{zGOtW1`If$c^DKVb;zaUDih-;MA36Wg=>nO^mc zo`5CKHcekwU|UQ-9zwFhsK6r>(HnjP%O^if;I96&?E9Ygb$H8x>jSPkg<+3h$m;(n z`=W4C*&3mIwW)Ax=dT)l9$7O@34WI>I*DCbMX?u4MdHsNZ@x&`2}lAHD)+}+MjJs& zCph3*dg=7vpWK#zMzm5mJc2}FR#B;Ldj z0ukmeeUR-VQ^;;VbNG>(u_MDYdH2aHMZ=cW_MtU4oZVB$w~-L6uLzorszHTlX5x*B zrq&n|#g&I#nA8cs;&Dm*>DOe}#=X1S4aIQ|PJYHJVpt?9(7jhiiUbf3^?-1B>L!Wh z;TF8R$lsU`VC7IRTsRTJaLs4EjsyGfni@4SN%ifaM$gTCucBrmPp8Zd&0iUQ8;L~K zLFZm76c{qcSdDAi%G7sk@nhl#K@@nHdDr{b#a7?e2*91X76eT$$~1~infVxQ^Pk*< z3ufDTK763+t41?<=W#|8-n*b#Bvcz*_0v6FCdiyXL(siX;Sc@eFL%h`nNu!jI_#Zeq6m32UX(4+RC=hq`J6b~ly?rSr;?GaII zawXG|3ar-AgB;_qY9Ke2A|T zAUBJFCEAOPV7~YA+Y1ZE2z)Ddic4GUqG)(H(ezye=Mw>az`Ceab1f_Wq8RaKbGy}M zXsyciw#*rNV8$S$F}I3y*xz$j>fG#-5E6#tMXZAzi* z%A#<`~_cj-u@FKKmb@y4G^_Rqzpm@BglAN1q0AVCJ_x)cl-&y$=C5w-`1iaZMs(rdq7NFIg-z}PtA~pA3t<-`MGT4} zcl3DDCyGfekT2?<6&gQcD>zIF*CL`7gL&b6d5+#M+%PI;yr0C zXU?JrNe&J$qQrxa4AI>v6H<;iC{{!?1gx5TuBDbQEtx-ZHSyC+_VPqG$CmlLk#{hP{FI5FWW zs>xoklmR8pPmonvxZlSmV5Q0n$Fl1q`5IR)MNom2zjEcpRuXWy{Ct_2p%5!72oKhK zXYl;RN;PB$IoS@@9Xv*g98SofIAuy~wh9On(AoU^-{*_CI%SzoNr{a70lJyAr>Bcb z3F98obcsfPIR!_T<$G~{Vz(KH{g;cDvc@uH_u|O_VN6Vj@URiF4p@!-RfbT*eU|NX zj8AiFtM);DSvDJL@yroV+{P(~(0kt81t>?s$p_?qk;JB3TZOrVUbxF-tm{1$SIaZ( zIE#Xu$DqqIp8X*O1;xW)9*F^x2<`b0?vHaHbDTAkwMBH}K^6+lBA#xX9!(x6M+-yg zty&y)6xOYUMu+&bn9=`HZOIDn;eDMwE1QT64O}=30Xdb|$@g{c{d2{wn=7Ij_n~j! zHi%&IT|8rmMml8pH<I=0DTSzea-_yz-=Qqh*sC*&=_j z7PIS({tY6@pl~rFMxrbCQ6A}pUbJdFE&;#Q;Gy#Y;ad^yyO56za;~3Fce^bsd)~=-jI--Okmt4#XNtjpK$+!1&uriA&Lx zxh90%$^`TD+3>L1%rhducb1RqEKwhuS_foOCMo|dUIy&#z__hT95IBxejn6`7TFEP zRqD<VX$uzb81bRqkJUTidwwS0 zLS`4crWn_rzwVsF9suNwkI1Fz%qUDL@Snt4xRLHndlsY0m5TGkVHzlEMFEinO$-3k z&%A(K+B?_}5~>{%|JyK_6ESeMiRdK|ODPR(goK1b1ezLw!%x&=%$~CigKS=P$79eRhlqjA;@Wb0x-6{IVl*D;VBzQq6P>qKNJcs5y|5$UIrw%E-u2QH~=h0$SN6#!*W- z^}^6uUtA|fl4l8~_s&krB|lHT(~&w}1^47_OzNDzWJ?}PH%?r6QyZcx#&%z>nSRq> z)qg6aqMa5>wW(+Y?V-GutvX^m-RCRM`0V=LTVDfaz*B~80`Re&$|TW)JOsFmYifd0 zoBcUlC2at($sXDMDQ!7|i#h)6kJLYQzbF}_4(nBR%&xw63xh#myOj&IRb+U~iooFU z>8yUX{IcOu$9G0gaV0tw8OU3*20Kx@Ym}uVj^a4MWcQolkfQgt(}93;u9R;5f-Fhg zRn2eDk>E|Wct;85qX!1(55UF!9oF|aP5FVuNW!@gW{(2Kq)0%m>PFbDqa(0jryJOB zkiuXn!dpRuO@uclVdo{iWVdw+0Y`Jh2O5eX(aK)OEb60brg85u3a^YC~hsX$8qPu1i1PS9uuNT|xEx%s7F z$%;U3FI=yB3h;{#jBEgvgaU9j-kX}B1rrbcIp=vOA)!*<2J0kDMXVq;@3is9tL zg(^#`P-oc(5k`nYGZ$84mK07vQM=UK=vRq1vDHN-NS zL#`9G3QBWU6WlSl)kwbWmkBaGW#Q<6q#cU&nsB6^(ql+2!PuL?11&!&-ui?rL$HE} z0*0wMsQ1njLoHI>(SAzQzS7``o@M_s7iz@MhositF_p9jI9ZU78~oD;2(sZPg5FaewHhEqt_W#EG#`rS1iq+ z$XnHbjSJ9-yKio7#**WZ+5wg7!%SGzU4ja(7lICPmF}X>She?iZJc8)8{|Y1rLwZY zT{0B{TVvcBeXrP3RSHCdFTL+$YNPa#=AuK-de`D_mq^p$|jlsBS< zQ4OK>T|aIq>d+RkqOFCqz?4H=)nH`symZ;zuQ}{4CToH`t0QTav&p@CN=#v=X&!$; z;Mf)h!N5=E4D5VS7zL!BOc-kgfF_yCrNmxQKgKtXe)zp3P}Tdl9)R$7HT^AxG$Lx) zti1jS&(9%&!ZLZr5vh=@3f~iJRMc;vwKEBkKzvP-#yP@7Do(tVoUPMS9~q2+uU8q- zDfY;N@JMK?RuEy%8pGE&dv&uo-vkubKyzdv2wPkEVlTn;?|_4-_c{5KuCrS={-PSA z7vWp;#E-oNG>NfIL!Jau&|)&filvUWyC?kjr~Y`Yl(0TII}wbjoIZ%tEh5^SD6iQ5 zrY)C)hr1T1_)>UM<{2Clienv*wkYx4?@K$M)tAroA#Nm^#Wxf$7x|l-efszJcX?l| z;2i&{R{$7O*ki%q#}($qb*Rp0Q=XKu{6Og-w{lzBn(ii9)#U=M&48|J5VXIbk(tg1 z1FvL4lzHmJsk?xy57-B%LC4m3Qu+h58QMPgbKlw!Z`<;QK>cDAgzjq${5FC8%c(Cr zGamvzGFK=aC7}HuX^bMp+>G~QNiOL$Vw!LPl&T8lrfq@9gdRtu&+GLrj<13}jH9c?-KAgsrb z!!IMWwgXAiOSrD=LU0uFL#*+7;>BOZ_7Vs1odk`vSM;)gaivEUzVF`U0N-kZ&e$;Ig}0}K7Z7c z@q3fgYqy2`#q$0}_2S-nQkYRcVDGn*>BkqNn1|B2XCPSeAvd_9*CXn>w=@LZ5adi6 zMS&BTm~c~4py{S1tr{`_vCH7= z+!vyi3vG|1N+4K%LE;QqKZg5m1K6-Q_~*BUFTHDFg)rO%8wZ0G9(od-lWc$s%PyY^ zzVoSbqpML$?N^GgCSGdpXzj@wS||axMV|9yKJR%L5YRU}|DxSplrDfvaor7t4vmNq zBc)ksPO&$`lNYM*kTdQF!dLXb&6>%Pugtefsgze@3&} zXH%`?Ndxtm17c=TB-+^vmcDmd<2}V=-{Bw5LXP=`(iEgOGE)NikNbsQ`mj zOoX%|CM%SYj&;o)svxy?85e}+>YngrkHF}aF-l@WQiob7D}o&7u+m(E_B9MyjXf-N zX|RUdpUD>gXn5+clK1eirWfu?My73DW~FLfzN&HeWW?Fvs#>mmOHI}AkW0Ao`ZCB3 z6*K!=@eP^yCED!f{snFO*-Ui9yUiT}d4O`wl?i^W$){(JrpMJVKJ|2hQHcqdYuKW- zHn#+!aJ`2Ii!BxdpcC_DIKm;YVkV+bk$O~J$A!L}X($@L1$1s&{N1PNFO45}Rp2wH zCzr{5dip0`{zv-m`H6|Tx&xT z`@|27kTd^=5tJbdJnGF4`#V${{P`C5;vZ%#ceK$f^hKJ6tdeA8EX4%!BXix(#N#cG zo5!=hrfPff4;9x_d2Z|J5)Nz`Hgheqc|c`^X!mkm7^JF+i(aLpFaq2cnjg$=16fqL zzcqFbHO3XV5WN=Z$3-r|=F<{}pqC+$m?ip30T#>##lo`I{8p;Z6FhD2)@)gYdDEn} zN)``zxtgZ_uq+wKkCKxys7gN9oO!=~{qfOIq^Dnnps|9MJ^kCai}6VnjdO+%Vq;Be zR`sU>Bfl%RX!tpOkuJR(`Z6roGAKJ~;j}Iw#8>E9Wthq9YBvi{`NQYEH)nLbR_*^B zo4`;j6IA*PgBVxz6|Cuay}OW!^V2QqH$sl1@PV2?Xj^p|poObfP)Gx@bN@V+$&H6XE24SYWG{)gWI>Y3A~V8ft#)nv zAzB)D86~l?eN<&!?fSXFZh~k%?0;X2oJyaD)LJ+JF9f}F|EIDdgYu|i+v|9BF2x%i zTB(CFF7i)VEjYgl1$PS7V?MD`MlEiHHUF*tBy$+ix$dGd1Hr9=Ja2FI*p=Ml_g{-A z-kzGlwog6idp?rc9*kktA(adzLbp zQXxZa|1uuvXvPn0XwCk(3VffAH6mZTpnXkjvhb1WW8D+39Rm|%?X64`5YQ+l6>(3# zs@kh$D>_m_3dhqlMQijOd_7QQl;KtX`{I&id!5!LxuS^pLI6>)MCt&Cn;%Sd@Yj8> zgv;N+uX`7W)qlMc+mkp$8)jzEqKVl@by@4%w>0c!cWl}2oE|7|<2OB3;vBo@Rg`4$ z=-!xFL3pQwMcsS^iZ55dE_xmpDILjkJm-f|RBR-{iDb3{+u79}-A{K%fSH<0mgOnr zEB-U0G6Nr$fAdhdA_iGXVVmLIyUg!bdn0=9@2)HTPYoB1AC-$Gy;Dg2UCMrg=c;)~---_JIBMX`d9Kw#1auluiEPi&J`ab0wBM&G@Ghy33;m${6Br zmY+?Q&i#foUvIip7LENhA2>Xdyu%kIrQc;WoGQ(-q0a+K^m9rm8Hb*Kmi1Enwa`XQ^tJ-B49JlU zd*GHPaZ@_*#5i<%rqNwvMJgNbBI}%T)Mu4DZ6d^!@5a^tKBkPGgrz|}<7ynghqZ$! z&mBORq_fQf@6y*b`B4mg9oZmn0vXBWCwOx9L$D|QW^eo}IgEX6{`a&qAuz0QRi(%& zDX$7u_@e)N0}weReI;Bf%Y58c>;b8rEFJ*FB5<(UbYv142EA!Nu*3EHCeM9XnT4 zb5h#n-4+b}`azj@!EqZ^o2Bv>>~ zK%gA&fho#nNwKuE2{qa*1QXJie&2_l6HfBffqxp&s#Z>SFG&@qn0I*ROTH^U9c zApR29zAd3B1S?jAlQi15roO4KF1@qbj-q-CjJ%E&tot!jqN8NpZLufRo2`uTF}TDt zFD3nw&2`$CEvjm&##S8E4>sBVJ;2G45pabB_5kAePfR_`RwRdi9^hq8~HI=6Jsc}FTUzCld z+3@h)cSAaov!b3K0_I6)cPA1m#X9T_g*;k6s$~n;m6e4oHPE>ggg<}!MYDI|&K@;e zu3VTD^RUD|*cnQJi0B0=DQWW=I^@sj{wn=7owyT$SR;7cBfa1n`nrYhID!vv=jM`o zEn>h4xHv?6RhX5HjOeuV6SFOdGDS^xXNqcfd`^76)$^DuNf&$mXnX^=!CNHiIJk!r z8x&-|?@;8an&%uigJ(T@XUyB_0oTZ(V%?9cM}HksWu40{BNZu8d4qDOYz#ba;7^)H zCsV2~?=ImgDbP62cLA8(ZT6LuQvka$EEU^T*o>|w<0JcvKWA&Qr^hej*>)la3hAQ6 zHgE@T{($`3SI?gv7qbJqn1?k(!C~$NxeULlyi6)MU60Y}J(;V5FDmr9$>yLmc{FoL-I;JYH2=Y;nl`5I%Qv`Q5(Y zqUZ}kr;CA8sS_QJ5CQ;4M+S6X09ykY*+GDe2d$64aw?WR>d`C_{Rmp}jeaxCpvjZG z37SknR<2>!P)fo7eb~o}7|#q&bkjvI$h8jVv{kTzbKPP&{RQY`M1D-Xu}Lrxcc+hg z6BNVfew*Rc^;n8^%>|kwA~ref^*~fTH*}MCdL|IMdjX;pra`X*-o`6|aWmtRK_J6K zQ<)b33Q#;{$xY|5)OeK}iI<(#`pBTi_govrrs57p24;}HU4r#COa1Noy|s0MEptcs z<-hj#Jwm>wsc-u-!w3hHCxO!;O+hZ_dANa`tZW`7zl}%Mhueg-1QgM>4dvb*uirh% z_)7lfBK`5sE_Yq=;;w~7TOdQNSz|PiljwofY$^e<&HQ`y?@ht!cmx02?hNp#X91}W z!;U#*2HS=aBonq+;)1dE!DX4wb0v_2{joLptuPePz(6)wK%ZYU+(84GL^n)WyRZN8 z@QWkLw{DN;P8Fqm78pl)^pg^zGagx4+O~DQZ zYz(>gDJBojUb&3kr#tB(F!c0_oxdm@`(|@l>dKb^t_X5WY9n`2w=;^RPZcGUyzYS=2~E0SzZW^RqfC9ttN!EcM#O~||T@F50C z-|7`$i!4gEY&{Og2xET$19d&?paN%|D9_-RHBkYeE3170yl>^{q<&E?0RH4<`a#{o z+?@9&bGyP7INx=Q2ug>`4fNckcVA8?E|xkQYAM}va&elyIN97m3p*Tt6staVMFZ0B z5UgSkmn%FUy}>*}ao4pgN$&$OPs!~JVXo! zQ$vp6{{T9D5s$8F-uBd)cr*JM=JrYBidLd$B0%oO;f`{<-~I&%PFbgB&PT;Iq)TRZ zvd6r4r#G%y%?PHoLadPD0TP-zxfGK}-Yv&}XNW!tkKEkif*z9?bo?|edS&~|yz_l) zO+rO#5K?UDEyLp?%)3UYUWFXE6O6rh^3@EGwS)B`5kn&?cQK>@K$A9iyZ|n2lmJ)U zR3PD*qD2A)bIQS7?GtLj0kw>)!Bn}Ds>M5Zs(6Yw?)Vk62+o_VV%C4aW8r1x%JZG| zt#k<4q0umrZyjEmp$e7TiQe8lP{Y1X(3xPcfle>4<8&-$Y(XrgTJ3x$4 z6pi_}-AUQ;Kf5?!%i3Vunz*jC>;Lat8w^`p!diU^D_{9O)@$f%(`$lTEM7Y% ze(cFzHlBT8s=|9-?xz#Y&+^J0-y_-r^| z#$+jc@1U)si>%WX#dws3QEa|hHr zeA@M@q!xB&W*JWxa+M*c=pfp{4nu?`l!D32lwx{={i;GI_OHQG*Z<_-u9BE)*0^VZ zu99U|nE$T&qNGULxl~<}6>Q>B_pUQGbllatiFbcJK@nw9O<0(8M2agZ>l! zKg}q(Rayl1#Usfp{C+;~XEcyD@D6pzchMVRSp`d1>=C@zXheG~e+D*K)GpTi9WGO= zdToNQA7{U4+M0S>Pg?ExaJlE1&7Vo_59jP8DwVDM%MSdOD4j8#iHsG3Qub1@r%KJ8 z&2wTOZpS>t3jVd8d}^2CzaV1~Tj(E8#PC(HrKJTr@HR7wTH;9NMA3JyFs9LCL8xWN z7w>gRRn?dXlZ;dbXzTED6ZObG{ub%N5}xMY9eaScm4eNS+1@kPLb+pO6rzv&QXoOI zNf5I~1>=bO|AVgAs?RkplWX`mNRLVn)OBCwH~mp+^lZBMr{E&9%62m zp{Kn2VgMKIG#q2NB?k2tu3YYp%B>NV^Rig@mPtZkk7p-;Dx9q#!#PVvdWV-h8|m{l z71vqmWZ&n7tYTjQ02vr2x)_czAG`CW<2Va^ej?yMhQ?J$7TsbHv}cBuU`{2;C(yN@ z+Pxr4E!T8mWMy0l-O{}pS&o?M`SJ+c<{Q(esqhQ%$MPAQ`pE%$ku9cyvwrdqk{yV48eeJ+meGT%Pz zaR0;?9XRx$JWma@<~1*+qJS3Dbth>Y1CP88XTPetdW+xcdi2H~8V!MTNmseo78Nm5 z=gzr#D&)w8jT)c%yS5mTv{E?uLh9PGLLB3G6_y3n`qW`-!DacL`6+247QD@m`e&Oj zx8!H*Q>``W3f`oJKRP6;T4~Enm;O~OXDXxcAT0j6qBQ04J5m+pqxXleh61e}%QTqj zrxvJ38jcy!tTY7xNV7YQjXFsTx%{Ty8MU~}9{j!1U1z>~blUD)wD(jvG-+05y_xJ} z6!bQEfRClz;|;b?5zs#_mYm>#cLfTZcXrKYbfI-yw)>4g0=J)MKUk({TtbJH%?jZ*Z1gs`VyMQ!1y<4l{~G?KdmqSQ2&~ zM-UwEb#q!Vq)zEf^(aWg)A3@y1HTAD7DOVaOKi{A(yd=&(GxSuXrQTG3QEd1cd|f` zfDSOccd=y`QTTu^iZKU!d;8`@D02Qw#NQjf_Zw&_4%!|AAjTKW;^NtJQ1;sl`+^(f z{}wK=zQS+Kr{5S^zPZnXuR2=d_x4fqLDyA9JMzxz6+XiQ39&kp`*(GgYznxsU;3%# zsb0={6{kO3o$h&C|M}}rY_Yp1ni)aEA2a5!hZ6FP#!6!N|D*#ZsSkpgo=AZu-99bZ zl=D?&>_rG~={F{hCU(bf_g=#@A>8h~f%`;OtkjEFV}s<+((4F~B&s>+&+1p4>5y$K- zsD2bk_t9DstFvu@6%{`0I#Ju?t8Wu5i~#;&co&sSBj}8Ti=e(u-lb3zcZ8;AQF9XW}IpQ$b{OcT)X&ym~*L#ATLd<5t-JR&EJ=*z#z2>|It zO(_;CqExDQmsNko6B`)oRIPC?$l)NBM(w}VHq1H%13bvX&a!u8hN<4an zwc5+gnts}K{BVp`Bt@Xr1mMm~p}fmua(2;b5-N*9711d`7z+h(pFARxPWxm2YQ?lY z<0l&9vOp%4U>6%@K}}Cj3ye?P?Y0MhcLVO2|7`gnccXSWpP3Ohuya+?V55#?PlyIw zvgzJ;;N58s2r?D0G!0fKpb;ASGU(U?RK9K_EX>l<-i;WtZk#U8&}ev+0=WL`;$O7^ z*Yn34{G0K255oI#`VB0-DC8fLhsT*H*T8w+>$=Yi8|3sYLF`K1c5?^+c&>Imc@scs zgR4X!#dmumZfF=F11rGSFpHj!-X>9q2?q$|q&Dcbmk&I$n<>$ptFx9i&a`3~dJB|P z+=+25?w_<~FtHS`0Jh@-5(Pxu<~YsOj(|02Y|6{_tg7elVb2g+u*}zUVZW1CG&cKK zwOVBJHE7E_1<`0TafEc+KoFY!6>ETVhgVYa`J134y%-$A;`PD!_ultH`d!0pYV-@_ ztj~U;Ck?a=8Hmq4jxi|zV`T&|S6B;)-zD8!JHTLh{%`XF0|AnRbME-9B*QXxe#V4U_Bj=U|gCpMfIb{FM)deZjAQ{R0{ zxq^K>H#P{<8zZ>rPpd_>?w>lU?^?0t0|XnuquEw97>q7iyXcD8_R=b(RdWbszPLTRT~-)D=71mT0%%micuJT%JumB|;kC0Q0*)-?sW zs_RVHuN`TY#2#PWKS$}0xg99kg#9uOS&PDT6ox;5<{uyC+)!0WtNWnu8t+7mhc*I8 z7HYy|x0O>m&dRW!56O2O@|k`Q$j-By0=R1Z}I)LqHk~o2RMV>5XvV$UuK9s zWLDQ2*EykS^^55o?xHuK^?4vK5Ot-Bwl zCsD}oi-O(Auw6N2i)M*}Ye`2+W$gL4a2$Hba+d;J#3*xNE>G~-aX0j;l$?T&e>QFM zdF72yH2ZNqo-L{|THmTBd*puaQkVBs zAZJ^KVQlF=q;UH%v3?d(@eW(keWjP*zeYz);$%*Q<#j@DP3Zeuw;6t8^JTw>sYqeM9+&WF5nTD_`(rzuGO>448NE^0 z_Srn}o{`L$35Yxmsi^14jNeonE{Rds@O1`Hfh#tYmiC3~QvM)0)n2#+(|yqfrE|LUTX=TgJsV^tsqb9IWfp>zu*aXI>Pu7_X7Y)aiczdNr z+-TemzMlzUPQViqQ;N0A34OEJ3$MNNu-~`fOuJNmK&TWdQaavtS+s*FtqUxKOuij~ zSz{wbhS;^HT{~kWqDWkUl4VEA-HEe3M6ItLcHH7j%s*6OTIWF&@VLJeI730nXVc^; zS8`-LtYVMXY=u?UL(atox(_1bcsWVFTda56%35it)9&KsmSA~3(&X{v5M^e;Ek?GJ z+7Fu{Yr|^oj$hR^>cJG0U-Yxg?}@@*4Rs?}xdsn| zNyr$K@uD3QD2t~O2ObgS66}8{>>uV?9$5 z=WAQ8ldAgx^yly-go4F{vO!*KLE6hNqF22taUd}7haKjCIMCiID|B#zCm2z+();Ai zeQ&DLZ^XxPFVo2oCw5Va)is+VhQ8ld{E?gbdcHsT{>_KfQS)krn(aq-yl{A4pgLba z7ma%ATJa1Jo7aAe->9D<(cx}j8%9%AI&hHdp9$?DC}KWqs*AzXc!`$A+9fJyLroK5L@>;QIK8>ulgywPleK9IMCzt*mmjakks)aU4fw?*J3KOKT8Ft!QO8=uG#qrxoK^(&0?f&oo zPYp4I3Mh+7OTQ@mj$wF3RQ6ml4gK3i!Ic$ucq0iacvsYY69Z0^M+^1J?`th^ynOhn z8oC~}Mzk}f_LLGqnybtbkiHifn}{M8q-Ju96KtxA)}3>$^#9_Y)^?NQS8=obz`mwd zvdd^v`@6w9@e3?MMoPk+HEVh`hKJ zr7*?C${GXuCN%*8y{R}0|2w$x4VMw3DkiWFNA8E2zsgOF6}q&#A~olGOwyo=F!*HQ z&k=5>o=VAvmpy6c}7++=pCR^~NRHQOQczl7c-=vtzNRXNtz zqegE(k_($)4~`Ph!+TZMQN_1^cAe+X$;VgrFDb0nLKS$^i$H;N+FN^{TWt1sqeHiW z@SBiH?&WX_gLY+QrC#cKUa~?7Yaf?Azva&`Ut`YNcAOsu5+sy9vx}(}vcrr_O$}pUjih8`%3NGr zyj)4CPEZkOF3MI|iG`-t?Z|oYWL|mLF?(tLp5F{}srLO{krQ9txdqE^>}?Xw6UD|K(o>oF$`%E7b51{uSyYVEYwmRq@|nLzX=!UeSZ?>pn0A2u z>ex7H{{Af3h+Uu;dL`LVuit zmgcek$mv(=vp_)oU;u>|YIsh4SMGl#Gy0FvSd7smi=uatOwBYv)Lo=-WT63nT_CQ{e`wRc7wrNHd6gql%qOt3M&{Z92ZzEj zDM(V=R%SMjJ;zjHQGI-b>M{L0ST_e{9^Y?G#4|qGIVdex@-4S1i+@|%@@jumU-%0Z zSEA^3>%ABWV&{1^WL{9NH=Yxr+*_jpipexE_|E`l{lV4g4my~!^1DUzzTutUf2>(Z zwjz0?|1S&R_9xe=h=wc%(iPS;iiMH#d?JuzRSeR5Q-GCU*|W8d#MP@+kaZoc*p?q( z%n@~``~YCa*RNl9aiygG{|*GTK2|6#%Cd~C};17=jDC$&|5(TRb90Ya4NwDI-n$YAX^P~cY%ApK0f zu4AP)ps|gMgG0R__S65D_yWcs5=8pgON#nZoVw+(*qF`_;2yL= zOu5!F!-T61H0INlXhRiR)FNDo7P@H^!voqMMXW6VwD`81pV?rI*scdWNa=fTc*bOZ(vnDPiam@ zasqR5*O`B0TR#X$)ujIYi7p{7k&I&oINNthp6L}2sZ6?@7bOEoV{{db?xEcPbHNwj zcyI;PkEqfgsK|;@ps3IR*{p(J0D1EIVb~+2tS@{j0^{dn$FDDg4!Qn)C_QQzv;-5d zc-Qr-mk58FT-8D{7<0yF>L_n@h3-eM3a*0cyg}hhD)`Qom>^Gy=)^?uL+ztkML3(9 znv{DTma)($ge2yaWd#K-{yjg%bb{-xK;?jr2i7fbAL(kwU(}?WUA`4p4pmD*P)R*t znGh05AH|o&ZS6xu3aF2zt4fsA8P>o0JV%hdo-qepztbd+103SjjNL`CvUUYER zYXq<0hq@IN5~4~Sr1Ul~qmN;%IF?GGL+pEA;y1g~>%TM55muxoL};OA9MhBz^Gj;EJDuJtl0_(jA*Y=q*8LeJ&zSTsIWK!PK=ACF1G zyO2B}E=bwK!*7|JnCSctw+-!K*H1l~dw9hkkNrObZ>p)&iQ;;Ug71$;iABUK;7~2_ zdoTO%L zbo$>k`V+7Jkwx*5&vQ5;XN*>5=g@R&z)R zzI7O2mQqYsR8?Kdj77D6!F^Y-qo?N~iWWgB)pVwXOkCl!2kn-sPM^5MEGG1(q17}Q zzad75gj#T$l0BOfW4YZJe6Tas{V3=SXvcE}cU>(Gxk?c_L}JGpQXfyjozgLS@sl|2 zJ&_(1o`tW1r4$az&^7xTwq1YXDtMnoTE6{n4PCyQ=V^Mn(*6!SF#>zpMGfIk;tKBcxriE&`Iu)bxHBc2 zz3i*jM-Zh}t;$N_7W-XM_+3N6?ig_zwQ;K?)B=2<%AwA6x^g!-N=G@dDoGSj97|Vb zjQTh<5DxhJ*Xv`f{RQ0y0nG<;W=T5An~J5EO)IvXOYSr4z~3i!qv7>LuYb@UOerFM z-^S-F^7jZ>pvOw)Mn)a$UG|t;QDiCt-iAulkf#1g_#u&a9tn&Z6G6;_pjG+I z<{p#jAyHELY;{eUt}yoX)fg8V;B zeFadI{TD9a(zP@w$kHucOTz+7NJ}XoT}p>^cetdq#FA11iqaqm2m(tADvh)>0uuLK z{QmdenVn&V8J73`#fj%U=Q)V5t&3IXulFClka{-Tp#1ykc=vrqtpg%V%}`tCAdpYT zU(PA|AIijvG2LcwczBB0m(J8u;xIO{+ahYPysuhwSO)7EK^pc{*$K5?}0C0 zis$A`D^D!n8Ut(oJdy@0()_TG_{xBqb^Fj#c-9;r`xPd#u>N=kO54x_eATegn$ zed1kn`B54n3`pKY%a^#x^jAM^GvbPH$-UUucdX5%yc5=d#``@dKsT3AfhUN4oFdtJV8tNM<0c!Z4ZI*_B9E9lLE(%P= zpzQt5u!`l4nK@eN8M7ZR)Xia`qkV;_Ee7qH!yqbM72BbE%Ipl`XOZp`J@+wR+%wi7 z6-gsxW=$P`B0rb%87KF;{11bRI<$+T$gp=qym2FvQhnezXAiUYq~C*BIPLRB)1a~0 z>h`=}79e@McVzd&Y92}yzN#XwdU(C-x1BixOj1~8BzyOgPFUshnAfC9q|(K50zduk zsRRR!+xkd8egR3W9MK`>QszG;hTAa=!&qVap3mErqqP+A*uZ#_m6ARTYTyFG=P>Rb z?moP?wY}e9Da|Gz)Anz(488b(qg|xLj8qA?zY{$6)#|o40%CwxoenO->rT zPWP)WZ9{WY2r%1JxwZEqBSBb>13pBUFPt%HyydLSVywUI=d9x`%8*+YAKV|jAo+SW zMQDb-Rs&AU)7lJ*p2hsGW2ObLCB6apXg$Gr_b@b3GI!sa&;le7MoSTI3FI>LVlus2 zqAwkyzTvO@@{hrg%`rLpB(-x^(Zz&erAfU1v_1wP9$D^Hum^U{5V$Y1Gl}I!^aO?O7Np8F1>dO5L4vVMe?1tlTD>UqpEGSr!ttd;rZ%m`{fu$it~GQ^TFO~* z{z|3S_L|Ir4(r3qWmD_hz9HtWnHKKW19|eI~Lecy6Nj z$Z!U0OoHYd5Pz2M$DHIU(YL|++-~>(po2QH(9_q<3=mdgg-rmpsT*3kLk9#xg_T7G zBv+Sje|0}CYknH7@civZd#q!sr-1*CzFRJcwG+Ax}9+w}jwr)41^7;fRgCo|J25 zWa~fbl21@r$NCSHp$)@R?$Vqo0_HW4#pC$ql0lrRYcekg#xw_{U8*VPrdGz;q0#Y~ zAu10~41*emPA=lD{4~?YtR^07S`iR`*rY3JMpSA4@&+COPakxt4&^m%iUs#iz0Sj|DH1pPZ6UwZyJZ^Ika9a(g>n*;Ag14mCt|3#VzZMt6eI-rK%2zAA6-4n zVHgG5x0wuNn5;|V^djh9YPc#}H| zT8{$6?z7Pm^m#CfPEED!(6limBJYrdcVNMBU-0$XE4mhCKN-se)V@_g{{UqbP|vTC zP?{J(h$KEfAz>4stcsqdHUHm??IoG9e!Za-j%6m$RkeuVIZ0BSJYN(8<~ks3l-wW1 zJ&!JV-1n%@t6~|v*7Wu5EUA_KNK@y3mGS?({MYgjZti%1fF}<$e`NISEx6wIooDp+ z%uWog13NfXusAywK`_ZRu`>bYP+_s{YyT(tR8S%r?Ru$NAU4(g{o#mHOf*3$_j}Q& zTeD4s*_lcL44Q*PmVayW6GqpnL3JSGCr{|}Uo6Lo*bUPH;@NnmNaO$=E`Wcf;-DKf zfdZ~J&`rQzv#S(CQYv?~sZGhqs4XfgdJJ@}jRRCS1s=m{gm*{K5`CXR$QL9 zJ?;J>enkwMix4S}+a9m~C4w{{2zw12SJSwc7WA1G$akJ9$=5Z9>cIDnF$8st%z1PC zBO7A0km!KcSxk}sqcy-8v5{b1dyDu{`g+syVDo)iw5ITXoN{)&vkWhEZ68S1lhJav zI>1;zR%$TTCP25(v>IdIO;JN@gAav=W2qSi0#fnEKyo(r9{Xh7f~QlRm2OOXdMN-w<5eZunfwSAyUH- z*qcAq?z4Y*i&g{*Cot&5Dt-T&gO-xdSqgscv06Vd(xobgj1e zFQ%mf{ShGOpN7_dHEpyb{`q1h=@`JNoEtTP?9dSyPvmuVGBbg$_YTRrEn)zj$%%k$ z-mZ^X-ak-wS_R|I&%wdL20G20_ouUv@fbtB8im+WID~84X)9IZSJ~NG6KS3WDJqufjnA<)JP>)p-)NuRA#}CG zk7jEVttY&<__h<9+Q-uJ-~mpI=zA0EXOi~MST!0_M7}gsON_7PNhQxqPU*`qR8469 z(Tv%=Gb|)nv83#n|3UyL)9mo#^W|`#-SlK3; zJ9mY0(V8#=`d)}&VCtYF0ISDyGJWUJ%0pli;DutKKQNela8p7^#FXWUl-7x#u}N!> zt3~mn#BVAa@F87g2Z=9MUl88koC)@t88~7l6k3S(G@_nR-<9=0P_3w~$gCo)YESOQ z`S06+FYFb@{Aj}ahlEusoHCZ=;L@dU?;B-jRIS~q$v!oWL1&aaOY`Y_p5m~*U^QQn zIiESR-NVNf%u}x2#!o8g%dJtE*dTcf5P4MSpZrM9WPhLo=6A&%!LUsJ)5>(f5r5`` z!WT$z7w``J)DNQXxws{e*pK98f7x{7y?qFI0t$qPdRRQtOx*yr6u9TtjDNiOTln{$ zW6j68Jh6+y6%b#w@nUnprzc4MI;mR~X6}7jU9nC8$YJwsXCBq2m zuXgTbLpYxMFLgRDM#zx8wkT$7N-WTrnU|-b_o1dUcHrcVSy1e?aR=5@UYa~Md&BK` zL-$W0aL0dcrSlie{_S%>xR>^1t1+@owzWDYx?%^MjYV-!k=aC}>_E0EYTd(Da z6`tzPoAqf<#;K-idc+J4vb1jIsXl7V7(x;vP+dLVcCQWFiwRt*ijXF`#Wrc*NjK0Z z=2I8W!Glq$?b!z@Gkny9lKEO}y9ujn%9xnoSYpi9pITzM4>x~26XO_Ow|nMLXatUB z!2B4UvM{Iz9L~JPSQ5b9J-R!1(u@b5nu?t#(&F>`6Tc8%Q)eyh)J@f1s3MT=+LEts zXt5^lXBK=mZ=3oX&P>PZq+maCvaP{=c<;eRv3vQK&mf6k zl8wAvd3>Ex!EGiy77E6T2wnF^29C0QQ3|otid}LAZ_!l|di-o*Ye>RU#Z!pDxgDwxj#$UvJPSCNLj` zHY86=`a-5)BPy2N8JGoEW5t`REUCe7)Eo7qYSxB3ofGGb&*7~i1F)W3=yNwVHnvzc z_x-yDP#j@J&*}T!0yjd>$Ns~F6A|X*h3| zv9a1&4NnWglcZza4#@@Q&NJ=H&d1kr;}a9@TW>MZZwnDnkndf`Y|WT=l6n(MeDBL8 zqeqTIk|-D>490$INEDcIR; z;kCX5i|&x+C@!%A`fh}wHyX?i2psXNrT2^>wkuK7(}sYYL#qQInA{Y6<-f+Z9J-za z7A%}^v?tyD@i7feDu#M*rPItmdSi$JI!)TPY)UFC*E@M+`5eoBIiksfcQH|ohRn_pukt&D`Tw&r&C%Qa2m3EwdM75wgGAD1Ee z3o&mUO{h4g=od?Eqc;uK1C}MeQjhUaB=dRo*jR&#@l8s|!`!p5LwmP)*XkOU4$P(>_Z^iYiO|dkQoGFZi;& zR66l+{m6ZNP}S}f8_EV|i#M!B6-Q6y{`b10oVebk#S8}I#^LgZWMkwEhP6+!11a*0 z=p4`aVh0v!4?3quvfsggc*8*T%5;VC&tpU^wzX~_OdZ+d@kL;9=^md=#U1p*XG6gH zCXqif8p8^jF!iBAySI(TsLzN@?%ryxqJyAHJVzB=SYMae6GDW?|`fmHmeq zJ_#GTMhD$KMP&y>x>iFH5U2rH7|ejxp-bXa7we;=_7op$P*wgn-RiGj<6WWK>=MJs z^6UGgq70>(;*5kl&EQ2!@{__xLPJ}eje%m9)J=t`7rer-DZYkHZu9$h)?kjHTo9hlWY4Wg_oj2hMEFGX4F zUhsOj63nC=oQ*uy98~E#g6egw>yRgOHN}#IBR_n|x;}Ni$&>{~-17tz15iV917(Dx zHko2@`Y<4PwtHUhf8s>IF2+jjl1PnSHY^GRKWcb*xQn>HKOuBKHTU?D zAG7H-Lsutu(4qU-SR>@^5`{ske)VwQAH}VB>{tDNDam~T4}vK~EW521Apqg0d?KZ7 z`5h?UL#IE%0XGAbd7y_ zzGwF^rzL!01(O27A{ZcuF;G(=gWf|ghu|ySyU*oHVu?+I2~Az=YdRA?(~GIk_}Lea zWqZP3tP#-?-Uh^i3!u$64HG#^^yvCU6_O#_4ljv@Lhkx!O!Yg?>B?EG}ip16WKtbMjy~Iyk|=@;LhB^o~1VD)0z1 zr`R+i^x}?v@T~O3wD&@xs*j9%Ob)4YQ{G2zvb!)h@RCA2^jG#h-K~OIyGGQvZ&|4? zjjrc)fCFcx)V1l=qFS+c?ayjz&sC1j4j2o*Z?WtWq)Q2WkQnSo?2 z&OWA99g44bYAbA~7KP+Bai~9epN2>3G{{?NiL#7p*)&&YRTMx0L9n+ncOt;=D1}uq zE7khkt$J~h=&LP+!?Wd$wAqrcDy79xc?A*Bf)v!T9I!{RW1yHsSekB~k0TRAj+&49 z6@xpGRb?KfgqmJ1WB`)4T8kN3-c(vdnXetk(^exGdJy+1v7 zYE<7TS?e!G`cigd7AskGzM>5Zq(F2EpS1E7v>8-O*FMXx3n z4W)0gJ=V_>NK!B0>O$vyk=QozNt4^~!8uHnWxRm91RjV9QXJvWz!l{F32^@&ZcR=cnTuuAKQ{(1oa`BY350uQf7ui@8S?)Eb% z5Hoix05LQ0d|l~y!xbA4N8;7Z@0^pSf8=ta3~o549nDR9MIPj%uj!{ zfR%s=nB%rp5jE?}Syr}yY4R`Ev7H-`%_PMPuZXzBXFsZeQPCHyk}5+X3KxR{cF|iT zf6?I*@Oi2SDtqXK*l%nP_%jYnJloU`xuD30ddDMO8rFbWBgH@ug7&E!J?^aoL@0ue z>HIIgvH6aN$K&`v`$Hk(k#=G)erijZLB`w_x+6$OmSYRwQM-ks{D{$?ZG}1lk{?1diMqUImIE>awY( z4h|78m+%cwCiPMsR)!JOT-({MH^vHTmg5Fg_T(w4oNAM&bR&6ZHMscBD#5$pJkrx} z`C>9oZf?D{{2iZGq2s|bsDuhG)yt^rFMjEtJD!=d@#T+nUfv`Bv<{CCs*lOzt+jKT z47VR#gx7rhbJ(26HSYj;FW}mo2IwFPsQVRBF_$v>xE5Y(X0z)W23Es_28N6c`essOu|A)rm}Gd0{*`C@Z-aU%!m{E?fJ!zYr^EmSc3)o~ z>+h8`F7~wMWw~rwH==*<>_iS~{l|+ZIw*qic_S|{w+c0x0ZL)WRpIpB zk%pvt&&inja%%6u|E^oO{4fmnV?_wiPQgCX>ufcCoIM&C4QSn|0(48v5bz6zNaFwP zE*557N2AoP4gKgrP}7M6rT@wAWruCS7RPhq`HXK^=wKGa0YMQ|sWraff^QW+%MTMX znmqsdtk<&qbAedrg$~v7RNJ}2z zpwzHXT7Dk8uk`K$Oah4>`BBk3Vl7=aewiK}y5($8cyE}|(g04)p2&hj)E4jV0^ZuM zXQ?v=8coIYlkSi?1-3BTHZWqFy^ZL zjvA?i_v(?F7^-MAhpDdj`SP>cob|?;caF6cGu5iL%j7zBj8nGDBOn&surv11z4vFq zA&N;4?2133O6Ii@!Mn~D5sTohpIyPg-lVfb#BCcI%Y&`iN|`P+uWi=3{OuORpi|za zvn&OK9!ujdu0yqlHlB+$bh)qf43*3<`N!jj4G+7ya`dhqZ%~5D@a(+`t_OA)RC1cJTMXX%g;gu4nVN-llk_& zrNM}hl}f6c`;a;aJq5-4L%%*l=08XEC=w_XC9LOH&yO`%8olbz5i=&4Graom@8#ZY-zxctdQnSOtRxav%nvM7 zJ$_`o;n~HwcluBPZh^9kL5`%EF0wendsyKU$vs>59JbM7Ys@PijT!E6CFmapQnooD zP070r#>hcsFIbw*2t-@9U!~&8S~#Way^9D6-ShlKQfLLq2RizBTBzLhCXI?%clL?CQ8}~Fan9=slFX~8<{W3JH6kl4As%#H@BR8g?v&-@f42skxufvYB z!!z`MUJ_g!(e!HL&gM`vyjStQ>H|&`zIsy+3>iL&eQ1bqh~e(9sbF=AP>|MiRSzJ1|uFk0rIn3ybUDcj|A`1}?a~ zOX?fTpQ%o2Jh(mtR`||eN_+_0{Z~c(kG?TdH99Or913}EQf&V&A|iPy*HxJjh>gE+ zs{fOkH0~>LZrL1h@=GSnm`@q%DlRI_jIKlqCEKz1@TTysN}Zf+q&uotAH;dVTL$kiH%*1H+ZUzZ9cjC@)t-_ zHm3GX5Y|@)aLm#Zpr|^d%P_B~n1HHn-);Ou^N3L1M^3_L+^1ZE|+DQbhlX zA)Fy1=s%N=fdv4p243K>rG@L=g$;mNQ93_wj)qq^?B)K_0VA=c_hajE|67wD*7^n? z?5X=TrDX-|eXJM1sKj+2T6%Yt{Kq9IbgV^ze0=oV$8? z`KfHpCSy^-Qwx0-7%mk~CIk@*Vr5ovxoFjcs6ml&2Ji^XVyeW=xmVNY&w6+rlRt;U zBpj4^D07E%RvS)mO!aiCK9Xk4gC20zF#8n&af)VAm&3Xw^lG(JPoNXKMi>rdtw}X8 zRh9J2Y#de0cx(f9XHxp)s<3@#JgFAX_Q=b@*;!vmtm_XKH4??~?~rqY$F6ND`?6i< z+p_NmpIKh)*vq{+gWir<02It8fFRBhpd|p326e0YnH(w1Tq-4G>~B&qlpdoaI=Bv& znbObWxs;jB=||#eDKdAZ69RpIy#1x5K8K8=B9!*^tI=2uo3bpmuIX>vk`e#np|Qy? z7MKc@On-cKEBI?`Ho_64wpI3#&2wW$>VENb@NN?WP-rP6JIx-Lv}f*1<%d6ukf#Uh zlu-5Znv8~*xSmh_{af*UYmaK>`-tavAgv$gh~r!|5gH#?i^RP?E5tJAc0^3By@0a9 zcwUfyf&wY77Z~AumyAsMVP5~4XWj2#{1^G=_9IEgZg2|5Y>dvMhip>@H%oPY^X=Jr zxOsRg-=a=J<7u&dy}S^vbEVnzj3dOi-X8-i4hH#E9MNU`)(TtZd^JrS89}SQ2gu#} zJ;Vm+7&6KUH2ekXG(dEB>#36QiQaPbXvA3P78B&r*Ye*doO#gx*|oPZ-u`^*+%VSI zt>RojQl#g)boWBidGpg7xmB@T686Y9f4L8NHe74Z?_LG(F*^Lkl?v3ZM7l$}HuI*Z zEtZ=}(J;}35+u(dN_*5ot6ay&$Ex-L+u5nup4 zkYE5Rn&p5hfZb?Rfmo`S+B{p5Q*E<;i`L$wn)N1C@~FioqJXKP=rg_-D!jA_JbtaI zM1G~|nFYURWX*YCUi-Z0dSV0zSv9(P34G+wKvoZ+0}~XZ(nmx=e_9txi;9Z&RiiWi zi$m19K8x?nOPs1EySzx^rOz+SoJsCbc;ykPF!DjL$&~ST!@-Z>&NBA)I9Z>A{H7|G zQu6M_83-&YS-*ookxBcN? z6;Ck~OV&AAgHkg$7J+GH{qT6?lZX`Hv^4oiTZK*34z^5Y2f2*<6{u42%52w@pf~a@ z>)7Gv5)n5KIg`Iokt^z^(XT+s`+2&_@}6$$qsGuYdVHqv@V3OfyK0#id04x( zf}yhCgEBJ>l&RO)7nDhYd-ecoL7l9rQf)}(h#SybQJz3HatFP%6DsoUf;A7ycYB2ci?7y6#@=6z>ixx#i(A)^?7NWM-hgV= zq`%5uZxlZ+qYwFTf;XEdR_cCNt}<|Hxkf#NZpH6MdCy~+_mDTC9c^m~$Kypt8+}Ur zTvm}t&(tO@XS*e?k7Ik2$IXa8T0u8uH#-s~>hdRs8*dv-gjE@i565lis~Kt_IA2WJ zzhLLhv*az`tI@F?K}kPT2o@Y9v~>dQ`-+JMOF+Hx2b73U6;2;-I5F|$3byL8e1zew zSNP&!D!Gu^W{#fz)k;fOy}|?o*XC{g58LG@oVtYc?Ol#3iP%)>zgILU3KNjk)WVM1 zD9Z-YGFCQv%`Xq-uUtNPBk$oG%0ujce?%?ZB)BI5MB|^93SR?5vCSzeYHJONN@sx7 zxFI=2!u&-&a617B#c|2?=jP_C;!_lK-v>wZkB>X-Hvs(qt0V6lb9De7Px*56MdVM+ zNTvzeH_yZN=HM#9&+dFRUE|?GA+aJ{NdLnZD(m`<;sLWBbmjo{sP>K@*AW? z2~C*72+Q?QNr1zcO9|nn#Rp{&e$8?=iCP;KED{0ri<67sVIwc5YWmjKF!EFY(S}Bf z6)9+UB{UdXEp%WwaXR~d9*_N20^y%O-wj~kQdES7;BZ$WjE!|N@wb`_WsWvx)Z@jO*3V^h)zPe<+13?G?M4)T|*Nk8CKx`@^CZ-p1)9XnE zP@+n483PgfEEY#W|Bw+RWmrc_u>W%V0;>C$QqEB=f5F4%>bC4H4Lj%Gq|JH-TAtrc7~fCf9%Tvd86wQ|-8ON{iaURz-hG`mJ51d749M7D7;gawStnRhPi@iy>jtB5C2r zWT3#8A*JW6K>`BqE5m@!=yNjmKxJIV$FK5Lv3=CSdN*GZ)f&L7+#6wvg{-oemo;qH z0kL!-jEkizoa|UBQcHb3MnF0tSuVi)*Aov@RrX8jZ?6;% z9zRu3l%^f`4x7kW+WVUJYjxVw(=#K6EhhTO9?LM2qhE0LWzT|XL+7U~i#A}6v1wZy zMcO@80Qf3#U&gAcsKDc5WB+r#$O+h6N@f8uF_k-a?tDr7YUio9+T|m`u;Vvui8)M}%2xs8{y&=OdY<5Y_{$*e2s|jonYghW6^u zUtiweY&dge>xmIaJ%Emrj)**hH+?!->Cb3X@XBJ}!1uX(0tFOenMzc{+kI(WtjOxW z?}`Ws6%(1lmK@{?vy4T^U7G|c81CM^`$I&!;T+Pk{Zk<|xC9FrppX z{hIBLCEu_@>yjDQCx1h%eeS4#2D^$kFS~21{e20h(8tST*eYSzT{nITu8Y!bNRs>n zm=f{siE0z5y8V%tS_(2tq|v?6X?jP!l&m+ z$ZxQ@8fA|keaUSduj633Cmci00D}o|kr7g@RHjJy7Zv`f)W&(yAvhW>6b~z@>EGB4`d;y| zZdL|mR_;3i_&GGqh*vqx__QWjZroNnnQL6K@M@T1Kgkg6iv95*#J05aprvmUSrTMM zM%p4CI@w(@ag&_>a^mKQ-uxKI%>w*uI69cqV=nZhp9z`_hHd+NQpv}+f`ufDX8FLN(jx%P-bKJd^ zmvUMb00DU=7f0pabrx{MdMLBLk^TAkad0jV{m;gdb<#FC&DWOLWBQ-ymEx%4UY>M#bBQf2xI1 zoh4g;*yPL)SrhM=Vru4lhEwy1AoE%AkB@H@FZdi75?an!7oFMpxH75v6ro*bfz1Nk zx8YUT3pVgXtDFTeLVOcwd}GH~y95TRYc$UTle4C$Kl2bK65dAHPsu?o(Jh;^SEHzc z#sJZmwOkctxkmG#ZiO@wZv6}ZZFHdH8T#6=Cwd*1x3VeSvqr%wqyV7zXL3sZ(L>cH z%#_Bl2sk#3CsKoE7cgMUDP55ZSmg><3i2>ao_mrBMm^ZLA)iM~6+LA=UN4L#<^AH! zOC5QgM&1Y6watDcs(mDIM>RC1O(!*4-qMmOTa#UH0`_T!$#LFP6ZG#H@1j#o!1M~|WNVv~B{y6%Ys%@{QQI%nx-*8n z+keQWTCG%!>jqbe>zDt5YWAxj7oK!<*Ui5nyTt<_$@sCfQUK3^jk{tw{N+OU^nPEO z!kesb5#5a&4M%3nik57HvMhu|%bkVm{^^sfkcB>Ah?&>Lu_JaU(&#q41R_j;W|=6j zsf-)vmL(S=d&~bwat(|ac`c{S2yn4BixVCcqnp&xV-r%N2cHVS0+rMYqz48DVxPTt z_?KG=*kXwV`|l@NyN)}nnsX$4+M2P*6ilc!^AI(I`re5%5h1{1l?oG%2x5)?T2P6= zkxUiRdN24tEda~J2cZ0xrax2+s~BF?(@w%xg(UQxXNuk--{hh@zZo3{G!%{~*W^BhX!-s|{Vdw4iUw)FYGbTE0J zQv`h{04@Y~nEl%eAM)PwwxkN36deux^$sNf#|mS0pa?Q{Jay7C zw#C1EYR_+abJS)LR6!6IUh#G0$XQIT4ske3+Y;)UURjB-$u1Q>d#8+jYlg@%0s1hr zBZ8HIT$SZRCf{lkk^ZNfa~fs5{rt4I^@?#m`!#=lYOOevSzCVO@lYTtxPz?XF>adw zqw_2xlm^p6*ir4Rh--mQJYU_N|LA>aM-Psz+`#yZUO)_h9Q1UkhsU8m&Uc`D(-Mh1 zqKMo6Rb9HBX|Cj#eJ!q&_7cM`T@w^7$)-EH9}3VyV4+V@1i4?fMU-*LKs|iqHIQ8@ zGKD%m=4$>o_{~yHchKp{2UhSGSEjMTzFdV%iogVk`=7VA(@#I%6HZM&RhfBg#etpaZ=(s zcDQBhH4%TWg14F1JSM?CER-=UGeyCFYe!f4X?+}{T2_b)Gu(kn8^x3_HI%`L9Whir z%O(Kt4W|`nF;z%l+)B+2Me0s%o0+p5T>i}bv8T9W$B{y4y661;w7kN4rjhKmaq<18 z`xczWZ3V*_C;N}_6h-o70zD`IM#TW-vXhacBiE)E)I;J4>1@{wQ|){K9b@8bV$(C> zYxbuYTnh5FOjCPe=LJe?yy$e&5RgRiX_aAz2?@7*dVXGI1&StsRVqsfy>ytlSdvXr za(i7>)0xDYG%l)9Tij9%^=)tQI9aZVU(~S>dR)^Zgaay&J(4xkH7Q@cdig5ko?zCm z^j3PSn@IDKlihfOtXRXba{Dovqs;S@x^=>7AFY+5bOCH!N1iKZDz%j3gO5X$mQ1hv za0Fd}(+_qdi!I+1YvV}cfCLkgM->9)PkolPk;h6?kL7;PyoKG5G&{TKN30(jfnQ_S zqzVUc4HB|RVE#bhk8q^6_cXM%Jvk^%Utw)0C%~%!WR=QNyU0OS8bHzI?b^NVd24Q{ zG@xMm_P*T3ziSf948;Gk{?}azP20yVspT|$K7>JjL+!!#05t^E-A$@tz|+FZ(p1-) z;rg^3Mf~olWX_k_XZowO|5;fGsp`Gh!G>x8bZV?y58j8-&Xz;(2*{bD@ZUUYQ(w1f|CM=xd`>wa0I!)KyA!8*BQk zbT4_QI6Oq}6_MY%0Dw*r3)sjQ%H`Nf9H#vh#+xkjfZcbYe5+W*0Nekm9e(2B!yiU3 z+f-aTZm4JTDaf1{z48iK_*5Nr*Lt!Uo)Q#X)l4^vd=~G!`7x!Q`^7m;cFyvfr9LPM zaJ@oqk>Rd>XT6xp4*NBv@1MIv;TI>OPcz|>n$1&hh(x-CvX zM&T`nQw4TQBpFY!2+V#^Kl+fhuSaFfRGFE9;FXI<9AV$1FXIR!qE2i+JtZ!>Y?k1;>eD#1+5d_N%7L?l!rs0W1cFArixf%^Cz2g5UwY&n+=cX@ zzY%h%p&0Wb%kQxk8fo*eh)jh}e}mr4P?#kBhNF?za+`pPYqKa@U{l0!$;%S*bUbXW zC=s&C#dvc7F1*(Mq+T_28nUbd!<1zIQLMs~xKAu%fl|0Qo2>3z^@3(_(5Ji-e^TXu ziDNu)O^MfQxPTdZ zr#Rh3EDV<+#4usAF-xWV{P_Joumg>KnHlV9i=dF=3!Ag(lrJVIV-iy*4rj2U^kG{K{u{j>vbalD!`>;#uorw0FJ8#jnX5^gfYdky_K906+nM>-#JZiN$IAsK{ zaN-=-dYe?&v=^WvJ+}(40~+Fh>6-FD2Mhk2JAB;=ByE5s%_r#nq|}JeCrVXyt69MG zE35-GjD_b1=N*w!+_5A}sfw*?6dwSL9&R2MFa6L1n6ryBm*fw46HW?*mzov%&mZOa zD*R3s~Kdo`LO44V{7C1WzFGAot5`K45197zN8V5V5v zqWr`BWzrG4g3b>XDdEhD4$1;Bf7zzns20XHc5#|E*}f<0VH(Yn6P_jS97iH~GBkjA zEb@TcO`Y|?Jaj=3_6#L)fETN_sVaA&doidTyqx?c-ix}?ezeS_(N5W)cJ6=psUAh> z24d2cpZzutrvO!x@cg`)uG@I=_aILFD7euaUdp^{ve zmbhR5*?s6#8|#rpDMtS{Bi3&(CU%0l$jx}tK=79 z8ns45HUVLT|3R%0|@N<39*>>nf#SmE$4q;d=}+X#?G z@o;f@U>0(9Vm`L2hAHTnsQmF_>rG{Sy#TdYi8eZ912R|fx({*5?Z-;C!Kt#FpKWI* zwpt$Lh!>?%E7EK8>ZC_NSKUe&0Hh_hV!xS3tjetAm1N-2f>_+%j{L8YK6c?C$Adog zX-D#1moLuE-KTqF|0#ZU&n- zWlx{$;9^y(FTVfoQUV;H@v$+TZ+O6%y!iU>#Z3Ptr%3WJwFHuzvzIp&wCff1tS^h| zHuit+1&ddN)%)+*WDaHWMQjMrxuH3IkWKt=^kUFizj#)&RM$$b>awp#;<{>Dg6I%? zQ;7Xabn(0B`*JwMI6Y$Soomyl*&Rw)Z^BVIGM@I%~Gq!M1cb6vX*5^arcY zukv&HZ8E3FdqyvBTls*RBnJZMGiR2TzTE$0$?yb_!y}|Vvv7^^7P$Kldk&RPrRe%u z#u^eKC(kOEte0PcB$;r8i;;E5t`DU4*;y7|k~h5#JjAm~Q#SjuB_K zyrR#wdjLgP)yKSo0o zzPuNFNt{t1Ff;J`SV`-r6YoE-S6I6(;~Y2Q7L;q(-n`J{aGc(>o(pO4{Y|@4Zf(Nr zbpL+gt6c4I&V{B_PHewH*>}SuT!qC%MpZ{t~^m^0b-PmzDm6D(qwh&J1c8Qluh1^qI@`FW_NrxD{|HIQ&MrGADU1^Z+ z?rx-0>ZUuTyIZ=uySqE3Ly$&F=|*XgPU(>E-1@xVUlt42xz4#}X3y-s=e;Hc4)7IY z0U+LFQm>C*a9$b)D98{aH8TsX*t1fP@tD<4qcp*XpUD7`r?~&eA;-pW=KcC4{ZZ|X z#iTYDa;moKjz{#?FI20|q4!{FS^6g&g&E_?yp2-MQf{O$)F)a|Ble}F6kY{cB8nw= zK&CA40z7g1hVR@;#nIEoYmcE2gKEs1Pn2rMFT-}0@qm1e%D;W$=M&oshmwwK$8W_~ z!Lpu^j-7Y`-pSetiH@rXOby+&6ai22_*Gp}8AO4jmN#Ff!uK~4v4vC~2Mw}p1K?_sBG zS_ae7nh!tm$))A&WAqoMvC~_P(0@^5F;W3uhl!F1=-MglS}S(=n`01imPS(lfXN_U zOs`Fq5h^S*!2BnKVYGhMx7u};rg5-j%4T!n#!9y(f$bN?L}VxtAxl0UrORbG?8*ez z*V`{Q1i7CqZ*eVscG>r~H>KnP6UezXJidEBB1BFzvZfAE1seX0dgw8p2kJj#w2nRl zsIl_!ySVF_-@s4|Xp{L!w+$L5)BIx4f#kWa_hGukGU39u6o=jR62wkXRE z>SdhFpd{Z)BNa%{9hm(7b#@sEqpdSSQ;c5C=h&Wppgi=@xzH3?LR9+Pcv`#ULvNip z)Z0I#G(hk`o!iVgt5r3FqVX%Y^vUBB`@r8)oPrwMPFo)o(A$QTBBb+2R8;DPIVCB} z(?owK8V$r_bX(!S(+JZb45;k;MEJuZjVD+`dhH`!XBp2(H&$-LnjFN$#d zBFFdLVt9<;^x}HOG|h?XV^4d zX&Ggr5%3;Ro2ir#J5=V8HZc?YO({)iPFHvFWnEb%Sd#RXZM}VGo2fFVKMJBfJ24&# zHJpz!=89SWz#Trg5CB+%tQB4wy2;G({$VQMlf0h4Zsft{CY(Ze-5}M{#$SETU$nmp zT~5U>`1<@Cu=)alHzJ2-SOY7Hz-cE`&L}o6L`q7Z=Q2*RWNH5hnABV|#Gv$iE#;Zf zh-sTPN6r~5dRo^wtR(3E4ZlO!##B=Omsxbma=MtC>G`YQWmA_J7wVt z#pu#lXHe)p(|x%BhEmuRtByFvCE0PWuwSZ8Gwo}LMx6WT3?cd{+8E)GJu^8j>; z%_d>Q@H4wJ~7-EJ)t*E7*>2a;kKEh`pUqz9z4JqP=?_d)+1Kl}O!$<$5_;7kkN zmtg=bd#SFkr+pwwxh|jRo^ZWMuG#iFz|%znfA--{GhFbVXimIDX04t6F;dRNhchsm zmWh3-pok0u?z~tx3g<9S)oLI#;tK@duU*R!9r;w)s0&4k%C9RC5c) z{X|ZMqAx_xusdo&R&b=jI6+Qj*V?M z){_;Q))Ns|bhrRXovk;_HIvWQSVA8VGT9p3I8Fek7F2u&3LffXxai4V&;NC zIumz|jPbDe1zThwHNHU%erG4}o(57htKE6aDl)nG6;%9{b3}M}g@6w%J?rN_?RJMe z3X#za7XWImBK?)EX5v6zna{slmVqjP;;gqjYsuxvrp5pL8K1?k;&PR52vukUfM z40I%6hM`iJcG!%dy_TOfKpKS`mTMU-;ETBP$zzW0+dOP8jOUncija5%ggod_prM%R zXtrY+uT00;SlRc@#2|V!QFx5jXk+^z@m=;KDmF*}+FMEm6}afE&@4{Z8Qr}hxinCR z_bzkoEj>2^z>sN2m)QBo>YU`SD{cQ%i?v(q*~Un_QN~}RZ-HD20p!v^5fC&LgX%03 zHWpT*%Xb;LNzzhu!Y3bOH#Ky_1X?xOrO3X*cmEZ1(By2?Gop-T$%6Wqz)l;+`g7lX zffnBfXKEsqVlpZsbi+~6b>tI(D;@b3;vqMQHW60x2~*b9>c|?Hvk5Bvmi_gy<5p7` zC#D8xZe#6&nc*Ar%?}=ZY_JgSvnl@R<|eIq7xs?DJ13GNl6bAkZ(|i{M0uOKOwZTe zh)+}?ogqA(8oRv~xuu2gWwCVY9nq3NovR5FsHAED(+sRViAhtv+xd zqSy_LqeA03yx5Kx^sYK)Ca=>G%>XjizN*CPpEQ$AbsA0880SGybzWe-Ai&H{O-rK> z;al~VM<+59_jk0@_LO)=!k`3~{osxuF9mP{TX4en5NrNVr?URyV;`4SSC4pFU!Lw# zzW6`ykgDnH=jjH5X*gq;P=7-f8X{!%!Nt}BX|zvcvr9uv+i5`hEf*I9oz5dbVR}Y3 zW4rg^{9d|=#xySKhgyMgN`}tR^B&P;ovHjg6nBlGdSD)id8%VNeGK)iyuGs{{T9ec%WFaD_mmv{P##q z7X*m1-9s>#F&Gr`_vNZ@T}kHuivDZiM;(z`Ij>p;KyYaQ3+!*CCEHXmlJ!ufN}GMW zIZj*Jcv;(^_d+rBIps>u*cH5Hww%f@1J6Om<-{KiJaGpy)_|pcgDfZ$2cJ!6Pn~`X9%Nn(w8*+TEv>(F9$i#-XUgd?U~3YxQY5Fx z&SS)N=LN3F8|qbKfGpzHBqg?g(2$HHLxB@c#Qiw`jBUelr#ghR!YaH=K`#U`OP*$7 zG1LCRA^Hbtz)ZZeauzw5oes!5M5v&LiQJoR+mg~=oS4=o95^g5RNPweJbR2oUZDmaxha&|Lc9;p1Q0IJD|ue-xvj+WqFc zP^}S--U7+2Q+;O~EG#iuj?qhB3DF&X#f5+bv$G~aE|FU20On8L4-#uo@HH_ll5q`Qc^ljg`j%jW>p13eYQFg@JL zQ4blxbS)5HY>89DvwJ(SxKTp-TLKW7rGBMLnB22`VZ1 zG}`jzi{QrKzmgh3+{kbaWJB#Xeboh393l(#&*%UMRb&9w)p#2VbOp6jp5vfuz?~E4 ze)h8p9t{IwW@T*HnM7g$=k8>s8Y|dN$PS6xi(d84U%=)NoDt3f?cnCM|jtNzNhq;y8Jo1aao+W=Bh#wH&gaU~l;|#FiLe^?R;V-YQhORfNPtnR-%| zxFy`vnm^JS(TmaKZSSFaUYZ;q6Mv-vA|TgQj4+d?crDNZQko;u|E}RZ2Jcl2>kM8+ zb4)33_+u)T9R7QXdb8MuO>LwIh#e@%=Zlm6Kv1$GUKUjtTE3=~6SDYQa4 z499?l4w)Bt)~Bwca|E1PmwR3LShXQsCAe>b4|(PKpwvr8(C@&mSsEwB{}l-*)evWsAk&Ez8jLWGhl!wibIY=xh`{chHzbZj0J z0t$OQieimxX}x%qJCLl@iiw9TMOB%s(*W}DOtoTlrxwuFD+Eg?;JdL586E_3qIz4W zOT-b;;1xxlaqRr^c4#J(ajL@RD5A1l7$>@(1a{583$CYuXk&qu9_yF)>zsK(@=R0! z>(M$E#+X^O$nIsr_=Y;bG+7oKTrjsm@A7K_&MG0^GnRGtJQsrsJrc7aA7vKPr_3G_Ie%5 zR+jf0>V2UVaWa<7d*~aq<&CRLhXK_A*^eH2QN{(LY^QYAt?unW*M(BVKHkcsH5Fi zxETCk5i&>&YBn?CV$8fh=xWKcm1Xr|48LDol;Mt8vu`qZ?8@5{US(un)bl?m0&Tpc zbj`0zDK9pu;!_uu7&Ftht%>g`^9T~!XNn)F)y>9B+h)jl(jiNtvcu=-VqWmCkZBZq zOVtb{4ZK(BU3J2(t&OvuI@Ara3O<0Y{#_;~c-IsEe4W>OXr6^zUHJJ3cuXB?kIA?3 zT5Mgj>9&@5N`{eS9@m19$=t{>SVP z0zOn)yvWHIjeU}nO`IPl%Ayt8pNwZL_g(a%gP|%KIIi~X`K8jV)1F~6{_zQc-xb@~f7I_H%=HKa5Ns&27z)iJ_iCF_6srU=XXt~|7AURm3 zm@GRxTQJ9j7`TaH2++tr4y#H_%-!5at|y9c-EPnJ-g_4{R0HDi^bsuCt;6Sn0JrK9 z;8#nxgcB^3{mQws$E>rz)Z_j6LxwW9vxJi)2gQH-42Ni!h>wWfPI^hj<(2I8{h~ZB zYjyEwudwlVM|ffgD6^kJD2Y+-+yUiISh)F5G+;tB4uy!pOdsG#M~21>$?y+`dr?w z8wrj{81dIrro`wStZ?Qd=IY1@btVPainuCUQ@yHo$ET)lNw=mxpYvGox&XXMK9`e{ zkH7Ap%rQ(fon4G3vbn~Od!EiH-xc6oLVV5=4;*wrlQx08qO_fVFN?8CRM~Rft~w$D zm$sV<+6#{moVi{<6rOW|Gbmh7iaMy-zZF(uwAD;>z~J#WBdJbm(d!%E*XN8W<>#ND zlZnuRMtdhRwckbHEWoWoa6pkw#sx_HBZp6I%f&g_vJeCGBVh4m&lreR6QSF7_e$O}z zD~?6H3>&6f@s8D}yXW!%Z}#R<-Fn*-4PKsXaVaTiD)-svwy_Ycn#fi_5~nLHmOrwl zNQ-*VAK;(~#sJfYL4jz#Aw~#vVnB-jpgTCoc=@Pkoxr#Sf}64)6>bB%QVgt9-r!OY zxG4`zynxgT1P`xC^>tPwrXAK6W3gwDSqdOMSz;H!Z$JSqPuQ2HDtlrZ`W!voC8tEE zhYoHDrVb9|<^47^a zH2?%=QUw|Paxb(qOO=TeMK_1|yGEDcAw-qtuG2!}FC#y!XMSRU1UjOO zz*%qwei_bZL?R5VTEit8eDPOny=_ymkcl(wZUiu5HdpZ(R&N-4ZzT+64&PHWvVUGN zAF!>&+RO|jlL1}|T>a^jv69TJ@s+dmbQ`6;YE=-DFL9s+Zxndc@i z@)5Ke*lMYeQpNG@3P<6E&2w^<0yU9eNJ4Q}Y8|$6)ze2ez@ z)HNe>oj5g6GbjeBe;L_G(4Dv4lHG9aDP>Y%WJg3z4d@GuSaKwAml>s=@qlpv!^$S# z`xhL@3Dm#gE0GR*c=+BJ-#?Q^lR6hJXWoU>T1b}r4+@)8w(9@1?b2?L@6d@;p8-i{ zv}g&?tnx-D!$kl)m4UZbwg8LZd+$mBV*OdqlLNT0mwx=muu_6d);U!EZJeC=%Sm?U zGn6Z!tR1wL6;CJk-r9_5c^MMOuF8pSjyWjbcYW%j*?L7N%3&Ak{trX`81k8Tk%%CtJ zFo9_`d3|^r*NwL=aZ)TTiJyz}o%d@m2-{3O10{8o9s$Kx!>x72YUS$VrE-MHKbIsC zECkQDDB_F8HfKVlt?Q^Byt@V1 zL>{P?eh@+WdOvdg(`9Og)&CN)y6O_u9MzM}9h|M8gQW9CYy_9^+71#YtHNcSW%;7F zELh`V<8d-I1Y;O}Gx8#=9_YT`rC5aQP(`C4jtYj5d#cA zvctsM6MfU;<6V3`r>SD62XlaDUCQ%T^ZK*IIy&MGIugH&V#6R<9tVtqj9}q2mxr`T zB)xyXv|DVRd{T?95BXB8lCH^I)V1W#M4(@U`oE)chg3l)w&NyT1AOwMmjNsYkcPd1 z#jvSM8Xxo}gdb-TQs-qbwk}qt{53f```b}ha71@f3ZAK6{@nFeJU{!^(STj3q(vct zej3&AZ-| zohkw=%ywP0&<+fipstMOj-x{L|KDIrgaRWn`rx9@(N9jz z=>i6z64Xxv4q_aSk4wFZV6z2XC*>!;fIotMuTL8lq|cjhHrU`c41#1~j1YFd^-sx3 zwmI^!?DeY2IjW-XNvWhg&!%4k$O(89_h?=h0LInoz%t1A^vtS!f07uLUr57BOHWx$ z?yW5Sk`dDCdf^n=6kWs@JIb5u=7^998JydXPEGelY}QgvKzg8CEnQz}-Pd5yY6@zq zAx!qp&;0ovy`vAcboo&LOSJ#}wD{MWwtHlnI7d3SbDy>P_v&F~jDZJzf4IN(+`cI@ z2kj;E&a;8-=fG0vaOmAt1BY4GDjoTaFK2|;(|}~?@^*7+<{i;wX(tOHEl{$*ht|MW zr+0}OK2MWDhKBoJ4*?57{sDIpyZ{EgFL`;;^Pni=0BARUpa42mLp57~gx~|SnEb<-HfO{xFN!S{xb@_S9FWzD*zfRK_Xxt={LGMZh-jnIIS~2bm_$Bhg zQonq?#$_Z_lA=7}W93MA-}r|_!YNYo5kGJ8rm^-{MP7<-Ap)Ihk?83V&$#H(-iW;Q zN1QxIVlvy#6m~?8d)QnL8|oWRZHm?Y6v?aluKb2B5h&7=BG>pr8Jrn%%0cY05|)(L zyx0Q#$4BN4Rf><^RJTSQo5n_l9j=tGIlpmK+_F|u0YE*e60m{q7LNaO_}h?nl+lF} z5AEq3m4rcf+i3M|#NIso=v@}z6xN#W#0fNR*dU#SDgZXiRh*668Q^d0dX5dON=oQa zoH`9Km@8n$o61^l_n+i{WRtR~l$<}(voK-ewRD0!x+&>IS|{)UA|{O-KvYSg#w8t5 z(026%3{1XQBd-n2iDhx*$Fu`?Y5~?#}$=Peyg$~>LeUABz^E^Fd9%fKa z`yYM`^fi0}5aexa)XWojs`Nbj`8vmgq}(CCBYq^gYN|9r-Cm zi9lHy2J#(Q;PEJ)NZv3DFj$B*9PIIX`Z%PAg?*0TkRs>vO68#kLl3>aG1N?eko5R`>cZL6X_nlyl5g36$ARMg3KIFT3 z%nU@sT;Bs$bzvo_#rnID{st?8Wj`da5v?8PhV~cCrm*9PYO1H5D%(T&m!UP7_wRpo z-@m(0m-}X865j7WWU_US`lR**vI%}D?Gl#(TOLeED8TVPKWJ6=I4C0shj3t8R` zV<$dtuUyM0R;Xuc1iH$8ya5DZ|B6mEYVc6P;g0|j${k+M{a03R1e7!4FHIH`1k{k* zjUyy3A7Z2%y%XCyG;QV*i1GZ;K)B=YI5KCQFYxZtLpQF1V z$g*on(hVbq0k`w1X2*rdT8O{QDxzvv3z>?u18$TtLOT?oa;M{lad+RIjCgdHnI{BTh8x8mpAVIbfFPj z?B->zq%im+`T>5^wjRwC>{?n_XvRk0e??Q&bet$Mbnkli1`))xs1bT`r*g`gfZe5< zlbS&snKgRFjg|u1Ji9oLn*V4GK;#X^U^|Mv#`SeEB=LM)^HEBkutr>czbHT*tnIt? zN!J^>Os-m(rCJ)}*rg=ke<>RUQVoOH{q)M;#vb70h>rL^>}8}>95nd9JT?s0@i_dH zu0AjV`nClTM>c}`gjl@p!6Lbs%B;0t_qS?vfjA9_XAFd{bkrr8RZ!d{nt6i<6yfeI z{(Pm@;7qfM7z0jC%1W7*QCJXg;c8E0%B|9%py`iUgUHqPt zyg+<(wW?V|Nzj zWsj`vpIe@+MzqJ7>eOm5oUI$Ru#%LnLS=W&tFdV-+?$&-&9*BcH%v~gE6Obcwt#t} z?hry0I7Q_JCgoH=yeh0rtFp9b`nfoiYX{GS5aI99#h z$)5{0f;fmDk7sOIUw+?;xn59*+JimZj$k~1BF*pk@#!EbFJX2ooiZgm72p8Cqm(EL zoSxa0D_}QMM7w9&#V`0ya;)3Btr=bB1=dd|+GgGpsTV-s2+<_l?=QowdbFh7lGyLi z%CRQ0Q0B=ZP5?06m^<-P-FM1MAO`_$hZgB!M`6Dmf0TSmEOLq&pm+2@^}F+^oh&#I zL{UkfL}X`BKSf_EQSj=~dzceVOM&_9?yXsBnb=q;kB&*jrE2lOG}V3Prf-2_zQozl z-uOTLHiv}KG&@Lqk(^+6%EG9HTAn^M~8R_K3F#h3oTvlbA)g;?q} zof?ZmQ|Y-J%-7H=Fox8;)IAWKqEBi8gzw?B!J#QHc)rM49QTKE`kEwBY~&J`x|>)s z6im`)p`~)kEz%|gdBX)B|Ko+AB|v*e_@rUU?t4%Ez9P!oH{Yl{?y94x4}SlZ1T}^c zcc%SQR|3iUgRpDu=}V@3iD13nWlH6Ey=%p8)L7Is{eM&LA@m^Y*Rsm@6GT!-Y~Xe< zFBRFqHlT=9bI$wY{Ui>I`q3BZ{D78r$UvMse7dOyHUUbb6bP<@CP~$feQtI@^@j7T zY(B1`UnJ$Hn*=<- zO?F(FDxcAj@UrQt(dOVv(Da#?i0I4AQ()ERmETvl>)AWucoi+NukZs>=r@AB#heAcfLRJfy5>T%bP3>Y`4-fMf<~rZExAn=nqRkyB z)YzuY%O*!UY&70xCIGy`cT$3* zVj&`2N3!`ce!~F99lI zJU9S%h(Kc8kQvVb{=X`K<$AaCy~adaKk(R4Eycu|OLgULE=zNS%oT$e1inH97OGB( zUJD;^r@wTHJ#N~eE@(she-)9y5r&qhB+x3`lop<2^!`#cTv2llaXvueU5Ezg$p4O!o zqjFcYtYN@W!v_g=lD8NU0?KAQo}WHz>rXa0?k~N)HQiTMRCVLBI@Wi&FIE+6k9kQJ zzw&)15!MNgQ60*PJzh`T;rMOFPku#_zSBNyAJRpG$QKXY4hlnHgP`ey&JRqDyB6*Y*bE64!iHZl&WT$ z;Wua3UEH7XjzC>^O`9dxq0J6YKyZQPN6SZT?k7(;Oee`VXDPV|SeW=On*4IyRPmJS zVp95E^>U@Z`*SJ77)z3t{6@kdg4w$$jb|=rb@~&_o2?F*sc3AWw+08GGv8nuLCTX& zBXG9!=wcy$H4!0shi9W>Bjn)j?b|8QU&w@Sw6IydDYPT9BAUQVm%GqhVFef`*(E*IFMeNj z$%4f>Y!4l58`o#L%y&?#JbJm;gIjdaWMP>G!)Ae33QY;2)MMGdtmMFfiDv~F^r$~5 zA0X#siN1@EgA`{UN3!)68KJCph=~zK+`yRyp8|;_tzu&LfLJ_dc+M@g+k2HksoACH z4OL-}!4&5p2jeN@G zb}=sS&XqY`Rq))RmD=&lLIajrtL3a!Zk@6dh4S5T{bZ!Z6QTpBy;>AbrVqXRTDcwBqWJnwMm}W)j z)mVxzmO=6DMyN#Rgq*-Qj1qwq2e<*ga&PV&eWsFn>TY}wGh37w^voNL0&m2kl-XO{ z38rYBO_ATNIVU%T1^1s2AiTqKPId+Pk1iIjHI%;B@*a&_{G8EJ{W*I)G)AkX zz}yKfGR|!+*}~~M)2AOAHvEmS6bh1u6KX6e4Js8`GMXyI z6JbskH!S6S8`O9Ymj{KApjW;jHu9?b;Zm3U+Vx642ET9huybX_K9S$^SNu-z70l<7lwC`CB7sauxPRr|3$@)TWrQ?8ZygKR*H6EL`~&_`-ZAw=~1Ek@KfR0e8xbNzXLIDj^kfsQN}bX z6IU*H?;wbYrvtr-s=*pxOT-^)QWQsVkem0>i;i5UtE;Y-_%`i?_KV0@Dd=Il?^j0_ zs~s;)T&+4InSo`4kzc5q=9q;D)-51O&=2fti`t^+SoZA&JJ%{?BV4w-2rw3V14kp< zrwwRR--11EH9!M~WC z#r;Id_=O5=-0LVk^eDW#h~0d6Fy6nUC0@Xf>w5!du?aCZBS*9??F!E$JxiUej0P$* zQ+b4GYlagS8~mxsn>ad0X~MRgGG7(*FS6*-P6N9~?CtH@p`IrM9u2w-C@Rb#<@-p2 zNVeD;s;Xp2qNAcd63CTR)yXj6eaAui^=7J-x%D>My^|Frzb})Zw4}|pt6~S3^`-U$ z3!0ir=GPa_S`xwz!(2y#f6!~wDa><>H_Ssm)_?1jt3UCKYb#jaYC-y9TbtSO-I`w! z?H*c3S9i>eeawSfX*(3gAN6yXJQ*8UIh!-2sS z911RQf`kT8s|`}q28WwC)M*7vTOxpf5Ec=xX5t#``GQasA5Zi4*apXTKMi@+xn&Wh zAI5q5e_j9uBv0zu(v$_la1v-0#&=uJtA3Tas{*wzV>CmOGs!E1bOJ&KKkN)FA#os! zRAxARC!o~3EGg6cuPO2{9ngYRkgUBp?QVER6Z0+eRp?VuPH(WeFLC&f#;Ypkyq(Cn z9c%T=p!A6NlF`r|$tmc?>6o19xSRV-y3x{dMU^!h{$6F^G4Pn63+3r>g7T(x*;19& zy0k^N97x5(xIVg@qyrcEHn_n6jV1Xi*_mGU=Z6rL9<1oV9_VxE45iLgO?e&BOnB&- z9$HYi#O&#Bhq8~_re#+}gJ!1$8oJz!#4VhjdH-SttN?^2KhiD7_ZL%0QN=eLQPp9( zW$ebuwb-&^sOY)iR8`Ap$z=hz~y$#xy(@u zHC7H!_+-zElubPgO0_|mVsEB%`np)rz-+&kSGcUOZ_p~}1ls2k=fn*u8s5i5ddwZq zu}GYbzDwduWMm|KbR65`o#&f!IOj@j}+vca)Se%Jn7-|*Hmwm9HDJU zf+TLSPFZlI8#_$NM)h_9h9$Q-9ylbf9H zL0WvSZXRzry5MCwBaJ?pq^F5~fybsPRc>)o3H|sSJnw7#kSIOEkf2f_U4vWKIv6wy z<*&hyS3L05j{X=&BF0bdS8SmQjP)X%XAA-_S)s*G{8?w~>r>&frZ`rW(=%igRk+DB zHgTpsAOXq}aeTXC4>;O0RQ2$Q(T(H8&jICYih1cEVE4jdp1+p3XP9?4f9>XH!uR_B zviS|#0ZvdZI_;Z9L<`1n69)9R;VfJ4=P~<3^S|usDIkUWj>k^nwR~c_IB_k$f^Owk z9+m%Az_7yul9H6mJ<|S0qU**a^gfP6=MS^D<+;!G)>gs^HTF5z)4}$o+zH9fe}~iu z4T}l>zRH`&{EQ((QUeVR3u}p1!_o%#HKDwAMXuj;-37j)7J$Bki~ zjodIob4a;}l08NK#l%FQzG?0A;7JOu!2w_p1mrz>JzB9M<>HS?(2PcOB!cK#Fge&r z52+i+6!gpv2MX>Fv++-wS{`}=ziq@4t+d2D7@dS$Ego$Ma=60(Tt39i|6I4z7|P!k zs&#Fk7$E%b%h5sr@J2(v_8BfF9yIhtq2ZDa?HZ+ZS4k}qcUnP_tMPVRMp>H!rdyjHpXk9bO?rI@w6} zcP&tot7@S*LyTL0Xxu4?7`x5GXYFX5R4X zz~PEIkyoc8MGgY#^57*LP9@4yabROT*-x`>68%$YPv<5hy9LawRe@41#%*KKRqPar z)JE^K&-K9mYpM9&e!<`c%B+!HqUJJ#J?Ym=axkg`qaabaK&kfN+xd@!S$&(dfp8x) z$;8>FD*376*@x;E8|Jr61X8mHQ$i6s*t~Gywq#{G)sm}-pF*U)VVag^bd`&8VTD@najM?Tqk+5TmbAB_DKKTN5<911YWo@x8y+6z7+yx2|&;CxH% z-hTO2+!*ty#<_3)AACi-fPO4$D0J#8G)FA2+p)& zD%MsMtx-lZ9L^&unkF=9ZV_rAvz*U2-7KkkN3xEqnU#3pvPMTxol|49c6?QIqQ|{T z5Emnl=~IQ02d;`wL==Bl=izKv7|ZT9m3*Hrv>2V57WNQ!EUswzD@b2~Yn@@a&KYCG zfGiCmvTJ&6=4lN(sN`nAA^95Qne$(^feIo(7g;_bLBK(>pblBTiLZGHMI?oGl{B(! z_9^w{6X+dWCXWGDtrm@rN`HtUOg>C~T3jB*>oh-`;G}kJ8d;>8e8WQA4jPX2aX9|1 zzNW{#v(Px(kI(fl_Zva>D;65cMvRd7#4gqw3X>r%e$to2q*WwJ?EI_>224tCXeKgW zZp`grG(yXj6Ec&qSRp@8jOT#Nq>ET5i4Hv|F9tL-->GJ27F6=n>v6ix;4YN4(E0E# zw9G39`2BEq&9?e>X2Nga>OZ`i)db7$+!=1P(l-G8EGjZFA2UI$V0!-S2vn-tHiAuS zq#ED&#Re$p?~;GexgD8&`{t{a%12kSLC}x=?k~;&$`#BY5@QYc8ts}lQ^;lWiz_uw z9?is=UZ_jbPBGBbQ0aVPTWBmMyw)%{&BWUYU-=-uSc)97@tb1#ng6lh)+dvBUs7pF zvrpZTFP4UY`yoN^me|{E7XsMjfX+S_i zf`BBZU|T1>btr9`hFawO#O$C=-B8oDK4~Qg$su287j%j4xSYHXaS{u30w`7?}hxXlFF1ZE>gkXl+9-rQnbx|X;P6rimX6l$Y&yJI9asQ ztLj+@4#s(T8bgjvI0i$^UsqXyuLzs{0j3Z5tLpyg{Gdc_(~6{F%ea!%4lJESZILM54|R;v6BRTn^^G+l1V6IN%)@Hj(mu3L4-;EfR()M<5#rx)>p7)$yK^L}IG2kxg*z~{P z_y*EIgI{C|ygaQFuxGIT+3`_66a05K*@GVA-Hq3rBBON(&GXMlFclaxWy3G%I(dDd z^)vO!*4T+^>x)C-N#T-k_Xshr=zmJ!O0BI4wd@$-p9<(m2zBDC$soXXdfqz8Dn+?v z!O)sP_RuIlto%sjcEJyeDVz+S*Q;kFF6IdT#&QwUT0cdV0fb|4mUy9_US6cbMjDsE z&^w=R%}*HFQ+yyFxTF0^ZVe?!Ng^H#ulb$*0vgS65c`7$)P|}9#Q$oNP1;S3GI2jY zAt69odj(Ynm5N%USy!Bznz~8*RR}0t`YfSumo#R6NJve5B=EL<2O^-8@%=>jF6iwx z2D94+&&M|NwG77bC7*V9pd%1*Rn*omO&ahq+=j3&1j=N4es?B|)6zNKDQov9D(hqL zn?Pn}<|f^lnHlKUFedpvTxit_retyz?@s=zvV+8>8Y}+K5y`_-m)@V(nHpMKHHJPM zrsHH^eES9z4S|mEuI}!tK0aMcM8Img*lT1ia%$iuxBzxoJ;H>EAwC{V!Ku?R{zyU>zgB!4< zCzO*Lk8oY^#@}?l;X8%Ld@yjJF zSbfh{B7@M3|Yd&>G562VNBwr*0wJ*6mndWy`T@AKy1I$lpWJ%=4 zEloz86Vl&EoEU7lX-MdZt>;}hFs)3CVTpN-Z=-}AKm-#7=~7APLBBEhH_1Lj_)|}> z8JDhh=Ty44caZXE&W#e16y0xOp0*Wj9k_QrJ#{rpwG=&^Ky(^I@W1-W3L&#o zL48qc-L_v@zQKsjp04E68H+u8#!?Ao$;3De`ZAxA7F%C3IVE}C+Es^qA}0HPi4^?x z%K;pLc`sl)>($)+Jm|dl^?jXwSNiSQn(fz)PoLuRea`qEve=9v^<5^oTAcS}Wfc|M zWJkb-ITACZnkxIDrz!D53&0jd-=SYX)B2=HL{BuYoy|f0q2mMl?0jPufH-7Me^Fyz zFb2hUzO7zw%|E(xY-=*8TzLordXgNHlQ*tvb5X2*i^Ibu0$PWg9G!Y>;M;3tg>itk z=tYfOmW_w>RefEH+{7k}Sz@g$*+{yfLsIo4p6ZV38gux6{m1{R$piT7Z?mEw(tpZJ zOBVu;8Y=#Pr3Sq|D_F3nuCH&O_6$g?LZ`sQRh>@PD&T9@>rj3iYL*@S`%DCo%mV?w zdZB)=O*P?hMQmHMbWB67_i%s0XMe8BP^T$y`kQ4dETC@{x$a@Ktv5jo3OtLwVM(p_ z_*C*sZ86$kgKwMBb^e30x^wy}wLS?~0iTFao`vvnI+TH)wO~)omF7>G;-)~i#@(-H z&MN=#&89hI9dRA|ui)rzWhN%P>YN&x$_b%rgz->c4FHz6uyi=;Na5b5-BLm!KHtCJBN_J_#djN_T;1h$L;bTi6uhV9rEJX%jes&Ek(@^lIQOajB z6$VzYgqxU}*3JG6O#R}8ave?&g}YSt^(<*|>&o8IlEbuWn(}&m9>sOtG2Ew2#T@3y z>1_@_{nJ0`zRRZ=;IjyUd0T%UCt|6_yb!5{?B*j}bat!RbrT-Bc`UQDFKaF?|H~j}QC=PtDh#-3Dh+Ed=WMzwW9lJIB z<4J@V(og|GaxeTPNlR0}tN(!IP$W8Mz#rdJa-gjyPbLkoj;CuDc)&2RSt*0$-b2qs za)vL=Sm^Vyx-_5roT2y@B}GHv2Y+W}=cM%ivGv~3bp2buu(EZw-aFBIiQWat)_d>0 zMK95}UV@0;x88*yI$Mh%(SzuO=rvjh-sN}Bx%WBGeg9$@jA5<4*8I*nKTVX}<;Vgh zTKKOV%FBp6Wb?TlN)vNsQ;79Z3MYqsv&R+JG1~n-M63b}*Y(qWkth0~ zwV7q8bFnHgyE0d+?$os6?}E0?-D!HP5B=dva0dR@HISu&0s;amKUPf_8?e@s$`%|b zS9OmG%!g<^+hW4l4%gt^>QjB+VDm-oYjFd@rwr;emJIQse|OqX@FFStjp-iDbp~)W z*h^8ODT<*9h?Tm(&<^;SDC_{FW6Z9s)Tw?x0PYls9y-CEuqSW%eXQ=a^uB0jVQ;^|{;Rz!SxvBNTW)UNaKoO?T3V%xPO;`c&k{ zxxp?Bi^)JZ5|aZ>zZ9spJ0mZG(rd9k6uoWNkIlmVWx&z!{v~UkqV-8oUi|94KuezF zg1u<@oFab)O~pbNUvT-)h=0Ayc*~%Ur*f=e=aHymV$RUo*)h@i-KQd^v7eZrmZa2E zmYwXJRZ=Qn4q5OjQ#|-*^S}P?pUJNcxv!_2$Ld94zdIXQ-neJV0l|k{>&u-&~WzSf)km7(E9Lm2GOuWDh*k49KDQIN zsv!Tz&yHy0Af@=GsnELMbPlpQe=`4tMvmOw&WtL#-mUZZL3z zl^g7fSGRnkjDZ2!#sEPc$Vkj6$_nvy%2!NOPW;r3r}iDdpSbE%&I7r~HgOI=P7`L_ zO8lj&V(DLcdM<86r~O>oZJ1$9VHj2tip5L>CI=zl}TNkGK~Km-7xn_)!~n?L*; zW~7{CNNn8XOufz3JK#hs1T^(O4!!J^N*W{@Sc>p9(*j3a)A+X;I1W$)9UIlz7_B@p z9lz>7=0fJAm2hIDT7kfn+Jd*+6_EtCrQi5g`~FoRpz^TQ1sSGkPg#)vaf^im-kk*g zG9y1BKNku1pOHUuKi(-%rSudIZoCQ<#r+>gNmiI2#*M-r0MM{mKd%KE93i(=&K2^7 z14STESx3<1&31ZPYN|C4#tAS7(ZSILeS9YqSgthR7$@kC^+!#t*tRn|hS>Fd4>5OP z<%r3T8MQrYFj)w!9R+U8N({RT5ui&%HDa(5wkiL7TikBKO!!VV&-pq;?s=y$9_sed zt9vL{TGJMDI6vGNi9_CiW)~&?ZTR=KcNxSlONcOUy^}u4GvWxkeD5Zbw&G8(4=o6Y zJiw`gsp*M1#GQ)$(+{nqulx0smwhe!l%|htP0PqrCGrY*gC4>*?YYnlN|2`^2ClmI63hjv#x)FO?CdL66f`cyExgac->; zJA3?s%lw)>{Q9zGK7){x45l%O`?Y4V$mV5%H+|VRlVjZ|_!qWO*0lMiX4BRiS zHYvGeD<#`;8F7#A*os#gMN6VO%UG)F&TGqP27SKt?4pkO($zM%wrczNwR+zn^>%Rr zRq`epVe^wE5&>x9zTofYn{r$E5*-QSG-2HfBLKgCmX1mK>WJBSyP-y~xIaoaufag0 zKBeI+!@_DO?#re-=!LB zM```83IUf=&*pK!Rw?a2xJX^Qc7v_u6$KASf4TTP!-8cQ><-Yij9XyotOc_tUEZV{{W&&hBjB$EBOsg(=V)O+Pv|>cax;)qgg3=mbbdrbi3_G~Eu%AXZJE z^{O8(p?RjjqGg6@!nm#7WAaPW)8(U*tq1e}Xup6L=Lx+BTIIcTeX}A!@RdnHNqIE= zHYuqbaFkMEr?mT)Gi1cpndZ+zh#rI_DOdB?Q33y(+Co%=A~~2TIjj^8*hYNRq_*JR!nD!(^dc!FBKeydilAi`L;z9wc&r&#cq(% zqH*M9xO-w`a)U+K`=XBaMQbTX4@Z(Axe!bwi5<sZ_W_;M1(JcEbObrgd-ANTMagPRUv{NpfH0sPNkRcf7xe6ouEn0#0xJe0 zY;;Dend!(B4g`p0YimJy^(OooT7=l%9i;_VaR@Wo!*-}I?%MGq5@eJ|H1{jnN@g9E zUBohY15g#1%e9lMQnf0~8D?G#GB1|?-UbgEkog=WzxW!ZY35O}4*J zPK~2Wl%BTXr1!52CG!OB8y^IwV=>9KTj=o%HvcGG-o{p|)Zn&fvbi$iMdv`Z?0d&S zggu2%HT;=e{68!1e_ipU^wY)e))N7drN?7qC$s`5oBmAdqPZIkz^JM zN29EEwD%$ZMnHm#%r;(2MPUqtW0pq+1dLCuTBifc9&Vg5wbE`S4o5jlu}&?DFR8+@ z`^WeXO_bnJwddmVBk&RWeoR)6LrzT6GlFKMnsEI2U?uD}Ec1o_QA@gKKBEd5PU%eH zwlTDN{ESzP2s;zp8kL^Kp?qOU|Lu%ac}sqyWc^Y=AT z^5ULGi~l<>w8_G27-`ef(=RS&)W?!3T~Ar}0No~pkXhxFmB${St-(d1zI<7?YEyiR z`udKkZ7K%h*v%pzJ$H|fi#&^597y=6(x@afRkeq0rqS_REyvSL^h8*ByDqY1^qm>M zhy20+e*7m(#maOEZBZ7`i4D*x%6467(=IN$s~uU&%@tg|8) z=D-v0sb|Cwlcp)z#9x-MeG z=G?fD*T4T?D>0&0K}*#<#0a=7@wdwRn|>;KR<|NZFnDTbwk`+PsQb*sRYKonw2dpN*$x4f*cI$8s>Na4qT z!>cQIOq?)7p8$>+nj%)Hoa})b2DOa#OsX~8b2g)^3T06GDo6NB#!vDvOgk<`1%;7c zQBjQvkx(i1*!eI69H95n(ATaT_%%qXmOdJNgP%%WzJUx?qGMnPKb8D>Vzi!5`Fd)d zA?zfpqZP$#B4^1MUGHD?BGrVSfe}qT{YJ9wiv0wIBJz1nExF`r$0>f`cv2{vI3MzusGe zKt#h_!L3jd;%fCym*Z6ms^toALG%#H__eC@HbT@u_B87LZnLi!Avw9?8*&UHrVJxd z_b$kw?-z6fXS8L&3+Wq#z_s&DumyS~`r1NEpzxxuNTlKm&Cx9dV2v%RP-cUpg4{kA zf=`AW0lP~MqDRKW5K8qu^QrS;X!EL47}4OYC8t;#A3pJ@;3=XnN=whOwe3S7 zeH9%GZKd+5pv@CH#W?k{SubRKy|GS@P=@?eB6?VWZ<14P3YtDR?aoumSXmW+SRC30 zmU-(eNa&dNqg4??Evk@l7)MMZ(4nCdO&N>A&l!Zov{S&SCi7gex6OYa~nr zTdr?%(ea{E0N=8|%v$Rc@IEhK^(zq6k^)3YR)(;GGKqb%r<(JB&ln4&*97Q!Q!j)m z$47vjK>PICOYC_FHM@Ycm|HqqfCUqkBRnkr;wT*z(B-!w53T7;R@!ePIS1{5_f1Lr)N`q>mpH39^m)YL*x%870URj3#SX~p#vRqglxwvc4EXrWuPK$X#TXULP$W_qj)uF1`p7aBNt#JWjZ;7~($|_WP-pP44 zX$}+@J1&R*&0!ey$mml*SbH9{vwlr+Ms4s^MR1_U$FKRbzaCR=B=yri_=~IN1FHq* zG^}pff~evm6PMh`%|61ogm^;P{BIdpc39CtkObvRWpfEfo*nWWU^eu0DDXSJgyU&U zq_kBu)|5d-(M%2|>3H!bQsg6QLjy)2XpipTN~5=mjQLW*IRJrd8;4(9JQqkAM6OO; zCR@Yfy}Woi6y9@z&Np`VX-b<_N)9wg&=Z5K39K7-zp(LXn7AFB)2Xy+c`9tcZiy@B z!2X6V&H|nB0Nu^9EJDoE%;XahZAxDeW%3CF-=+)OkJ-hSI7&URtvD>syID*QDx$c) zmccOwRJkH|bfPkOYimYRk0D}UfK}42P%8_4_+7}2>+~erEs~F^?Cv%#s9abu$Psd- zD66PIyN??<*{nD1tnBG2=;-LM(bxpe^&COR_hU_?g#r6)0p}wuObNXYmmdJTi!Rs> z0Vg&5T4iM^h>v%M*@UwI_t-^`Uq?t6QdZ3J_PzD{vT%CbzWhL9oL78M!lltvDUX68 zSzu0x`TXXW87eB`B~@IURZm+hiXu-tv5LhyhRhzUfbnenSMmr4^4dY)3=%-!De(GX zuMVb_QhCa?cU|d7PofgFN+sbr2Be53Pde|-pFdrbekZN7xR(}1uX2ki3KOdp|6Ks( zg6SQ+VG!U zggUyqOvX2p^0=bGmGyOXW%h-^8A5<8vO+EY<5Ck~J2u0VX(IQOq-O%|pPQa$ASOct zQ_%&1X(U*Q>8rz)XoGdwm1aNjWROV!<`CDrfM+5p;3)dd3Z=Y`LsT@IgvX4t&~$R5 zXA2OzSCE?dL*YQ)?Oir!CK|>q14-6!&Qk8TIjm{P_lA8MOIimW*1#YVWkqk^@iH6* znFz;2vymoqnEXFD$A6k0030YAz-uEg3p!J{1w3_E?bW}-`<*{@D?TNo5|fZzWf$P0 z;zGGMAnOtDQ=7XxbcAj%XJU3&&ze{vJBrL2f!>EITt3%%B{O9>I+`Wm-Jojf$VNb)O z6NQ0P*PmaxAKhN!4#hJ*GTVU%aey zoqP3ff5pVF-5w5dR_X9^kb7^ zslP~>`Oh={pKh4NLh|FfqVtFLUljc^1HsoTN@S&EFGT~dIPU>CAS2=zV8kMg`jHLM zP}sz})~wqIjF(99e6&3A_s&FS6EfcyC=|Qv>yrVVr_2B|-w8SeQlWnOB&y}cB$L^% zmiOe;O27H3C%nIxQ&(P=XMl}bFH6a=F-w-;`ip7Ly?l=28rcT^V~F9om-wl%B|?}L z)r{P-U)SLGh@>wuIJ#K^lG|?5p@~V%r(#fxA%mDGA|258^(y%8YQ%86AvVIIf%Mu8 z+Zzp{0t)Vou%$dBTSvl2QU}E9boaib#VRs4!(o#3;5o*8v838R55ms8vD}tvpfsKpnTeZKz^TXx#)F%s{Dxv|| zQsWk<^Z^lE4F~`RDW0=(ag{v{@bw-0>@}Wm23kQKQ4DRxRsF>lVJPyi%3 zpVSYIn>;qmHey|<5>*ku|NCPAUJWV4_R>e;gFi+=$_eew^xQ)dB;m^)? zt>jnFTmk!}J5aO?*nnGv5^r>5r04{Q^6-AVzgA`tY4t-F266T&!F0RYdf)s)r*>_= z^pFsiOxD{XUc_MrltWzg4#mlz^2?LB3027)!C?Rp0g8`CV$cC4US%}2M6|v{)e|l1 zH&q;+9n$GGZMVZHitKlsj?(&j8VCKpK<9{aOz(ajp{}P#a=(Ea`i^UuGoTH==j{9D zowtWfE=H@8e-d=PfSO`oy|aIpOW!-xC<|IA4I%!!HJc zHh2VwOQzwt7nOm0Zk12}LhU7(djEWL1NNNzWL60>6>BVS+xHK0oy!5 z1fpz#wzalvn(O%J#=iww|97yxDF_>epnY$?y7CPG#QWQ5eN1Qkej8DwSAY6z&jUV2 zenhTEJe|g-9pW6v5YZN?Bv|i2aO@)!KscNJQ24xR`kbE^BhR6_F`}TP zk=4hcRktS8MsC{Zu3Pjf4gYk)Q$pi$WLt)DE!ZD_eo$Nj<)aC-XO)Dpo{dE4{# zDn|>zt&=v-hqccm-v|+x1M9jcY;fxpR++85j%T?l#p(t*;#b}nXNu)V`cA(SU*kOs z{_HKf&Q_#9n;}twq@e4@ht*D0YH2~k-E@f4JaEKd!V50V6Y=QRy1&?)se-}aA6?y0FnI<^ zH6fH+C(3&W3F-!JbQ#~X1?)@ZP$xcc-ZKLHQ>M$ZZW8Pv5LWc)FLn-;_u|i=4L{R{ z0`_2O*yyKu00~tMKQH6IR|>pP#AoY`(;IG~ca--)g((NLa7H=FKWuMo?CeIf{ho4! zvAEm9ii>YaOL2Ni`IJupKb2y@S}PJrmg4lOChmGRfOcw;($OKd*y;Bni2n1sinq6* zY8J0-4{Ma(oNr44*;sX^Tsv5twBih{>$ zh?8S2L#y}RJq|fDv7y6D*WxI4CmhI897ULSnk2veu)J*2zs_&+d;O2a-t9BME|c&M|&iSNrmmAV1l z_j)i`V}!ZmY<)-@3;2>VTK&()2i4>fPEDih)NWRDgm=LjP_xGv{Sze8? z?Jz4IqIrz@Jciw-f7r1p?_LwGiwd#NrEZG+7trx^o+HY^t;+_Q9cOG#EwlOtxMWmS z6U&r=4bk)^L6T({RXkePDPvpFE!@Ri{7d3jGLx7Z)IkUMp+FlFTzLZk3LLZh?5axr zP@sgpmvwF<2obl|s|%kkFvIz1B>XkN85UJX9Q!iT-P=a{2|PWpwfkegX6JeC&Tqrm zYJ)Rv*Q_DSRWM=Cg{7DRUPJC6uw))mNh~?rt+PZh5(Yel~js? zNYh)?-Vz~&cSh-?@D-IQyUHUipGIk=f90i;ee2>;lP#U0c%|X@+cl*^A1QRq<71bK z+xy%qyLr_8$`1`r9)Y^}EkxAiZAeS9wt6B4MdypCY0?nKFu4tuwGNEO3xsvwd}mNiUP^;|!6IUjKbIOgnQD`WX0zcD_P4;#i~M z(bPcwsay74irYZ|*?dO3Ry4qVODU{oo;SP%Br&&cMR^WnTUf>D@xJQImMJO*d9>Y% zlfOvcPAC&9kL3uCIAA%t+>PJF%)oPB0`)x)cOKQP(J;RdeK%I?TGhfz26F(;BJIFb zB~>u-1QYr$cI(SO{HXF$+sr!}I{uH$EN^DX#s@*D^oYw7v-hbi$vFLlUrjMtk@iKk zxR9$omdTiTW6=TaRGWG^P%phHai6<4g@r|;OJMt9@!rj2qEhhgh9&F0BjxG#9~nSE zMWs2l`e4WIG_zX19UF0 zga;>m7@0MERl0orWa{UcOoi=NlxD&VqI=@Yx22=m z(xE*@K5dcH2&tk#8VmhQ7x1Ld&Nm_4hOJLQxVta3VqA1{peDsUqbo$r7VAa_$|+$F zH9Nc#+Wn(**>9=E_1MQOM6nV`OlE+F(l^6ZgZ2cUBvn^cmjp!~gOhx)qHgB)ProiY z{$?wAeabos2B7uzitMMCs5SpUQ?EC%mj!?KEbNXMkeH5WPvp}eAo(ZRJRv^Upwpm< z)QF&oANtSzw@212cqi*xS0VG}q1{1^Q3p(y;m4I5WH!UYrqIx}&f|^*U^qK$%Cu_> zg~NJ!K~z1!;Z9(}xr_3eWZjy4j&U(Nh?<(3fA{y}--*XBGOM>i$FD=Le%N1(i=TXP z85tP?N2qidb65DO!D?HwSH%L;J_1Y_VWGY^sHX8-tjKljWbz;Kg>aX@sf?8a`yFi? z2Y0}wJv&oH;IsYI@qk^_cKF^OL4KZ*^LsGJHViE4cAJI3NxCjqwRiJcGqt6YSJ%hi zgum=tTclP6chM=GMtzkp+8seg?33Fb=L+0{*93gYq+PC(LW14qW(_ytY3r2)fQ%>G z>{0ow!7!+O>AbSGHlZcld?fT?Cv+NUpY8pANgZcrox;LVYa|w#kT3?ilL!oqf{vWc zOMtaOi?Z6=g`VI}Z#E2D)J1ooc^lgSNjzRC0WOKJ;}W5sipexbS{@BFnh-Pt3yXIl zfH$nr8$L2zCY&^JJl4SDwx8g3n#xCusIWMuECi%Kfcqa0a^^r?rWwpYtC8IuJJ`z> zjx)-}nz(+Bh2)nZ|7vfN3rRHdc^Xz(xabuU@a58W_J`|HhjPEihPn5PW*y$G1Tt)3 zu_Lj;|H?oRYYg>C#r(ff-VdzQL}G1P9mgF9yMS+FI!^-GV05o7w$_Cb-q`wkW$qNI z!h95>Lv_jet*Gd_580{pX+`S4XSLOTuu|d8{s9<|X*@$&E#YGL^P9cED+8b^Y0w`D zvDS#B+7???7QCBKG@1sR$IB0)N7KrkOvoQD65uS`_BIcR+xB~NnyHfP&JYMQajVeO zY1^}$ESICsr4pq3yuI*OqO%LrvQd9V(1kOhocDYa<@e_CTui1*4syPp*5Uk-WqRBzEP^|iK*rT^3U%26qp06TP@x(mZX;{nOkft zD+$a7I1Y?V);wN@$C0+C|F6aj@dOV*8DyVh@a&R-hS7NX{Y8gmP|WJj`sn-szaYqK zL#tmufI?n&fbvyFJ`eO&V&;zLlg;TFCJ{E-WFfE?`)d(~+=PceffT{Jir3}@A8XND z|G3}#aL`+esx?XS^&wZ(i+RO!j4OjY%>T#dT5W8i%jcCPT;`qNRusS0Hx*14%&4Z^ zU8i!c1ZLOthz6omuLXzYMcFgT-f_3SKpVFmE&#<$^&zo>LkeOm@)<=TXw?of{NV!tc*I%PuUNdQ6 z!ZA1XdvDWKu7Hz#sRn{3@z}V?go3uO||?7 zNN##M6W1#|0jvzJ2Hrj`=k4Of@SWnoR_s#0E1PpV*Y9XUyGM{%16#wfXx$t75g0xWF(;$5M*34KSJjhPO1v zdS|qs@bZjEeT4Lgm#KUr;1_^;O@19WEt?FI$~&9FTt@D@MEpzsGloK7X+2XEFUV(u zDdcFeMXN8&_m9@YS#qd>U1Bi}t01h><3bKqF)KUnN4f>NY=diC2wX+Q55&y{Ti5Uq z;T@Ry0ZR}Evh8<3Y3UJ=y)TQ1e3a`_RWDeX7CTi1rQs3sH4hG%W$CIHQ&$4bF2^s) zpBV+do&N?pU0(zie!{%UOY^?ee=Cn3ShAVvj~wG`Q-|a>zGc|_fyAsa zuCJl^(estxkUzd>RE~^Mv`p*50fb$$@jn!$lMmYKD(b3UXl9l}S z5J5&31SaF*y{7BfFu2~WB^JcMNqtIH;+iT{$8lh!pZ4a>n|hl@NJJcRTV_NAqb#LI z;qV(Qk}~Am@HIJu5los@+Y!&+nmgUl~82;b5yyU@=we~ z7ObBJEy5B?0T+|xoahA&_n1->#wrnPzLcF_HyYyQTns`RyjC0}CbEAZ7JA;8jL^)EE?~=pJ)4*39$bQ(t z?KTaW!{&F*J$b=~_Rh{Vy6#G|DAH5og4<+SvVXDL0yM(cr;k~wt7~e?`im+nUy_`C z{}9rmE=*iaV-}DcHa9Ap?liGz`Jv&pf5ond&qbCs%mo;U7>5vC^i)nkRI7o_?|T;X zAq{g2r+?Tw?0$&ECtlUbsq<~yuH%WYEWZ(?T9JLuU?x9T?93Lj@FMp!z`micqi#?s z5Gj-!_#@T`-5Q{lzUjlY2{UyiCL4r<5V61Zm*Jp}TGpZK3!!WLjgkJ_$Am&Y+PBV| zwnfGovuW}zMDMcCHHqM;V-YPuTEd~26T9uF89J6Ewn%45sXQ$&tZq;js~3pN-9xRa zu7-dweJtmM6@-;>NI9G@JLFUcF<{!VtrkW31pf@RE|~66&lGh zh@~gq#XZp24FZx+TUeN4p7#@OKF}rQn!I$L;H8n8)m1US80FDu*{eZ&9`V z>gg+p{rO~7mFskPzpRh@0O#YcE4ySTbixelA903qeRL*_7-J}`6rQPGtn3n_CFlRe zEdJN=2`KXswZ#^kziKSSAR@5Rq``Z6&8kgPa@~4RdrEL*RM+#S12oM+Rc2M?S{oVAu$Wfnu1s3=GbOr zh^qpF(r|bOW1q}5kif%nLZ0Ol-nU`?ca!u2=*GoH8IXB^C4X-w^16ml8@rQTvydd} z+y`LC<(L?IPj(TLPJ^1jcY~`h-jD-ludZlTU8z)yMRE$)wdt zS4%edv9u^{QS6%@QuR?iH{$?9nOvU>KmqSXh>{l>9`GQhulSD&=|3F~Tpp>``xGxv z_~X%P_fj<(c3PX*9o}&@kQ`mrvnml5(^m-W8bw=t;a@d@Mn)ewTow~l;@e_YgZevq z7<6bTTG?-L789pJ7p26}$teRu(=hgHPR;1?$E)gk;oSbJXy!qM zslSeYoKiC*vSb2%Fmz9hlrGU)r$uM}^yzxH?`_cl^Cqlv3*MW7cE)yI>8x+!5k<8g z{43kuCIl1;WCU~`eKzm!MnJ|a7qgcJu7uQG%%sE%WaYhXchkCOvSZ-8ITfuLD zvH2bCR-Xqe%j@n-J#t!zd6JnS!U2|gDNn0neOiSNTh;9*CoSr{IS^*2%x?gq>xqy= zGzTqx?tTQ;%c-wIe&?=z2@HjGAl+>9ggR=9z@td(1(XQcfe{7*cnova8MQ#Ja7Z}Y zXgt1&5=sKQBr27DT`NQDc4kz0zI359P5stOx3Z>?%AZpyQ+C%cyBkR@M&6~oQ!ApE z=MZ|g*{A%!8%@*<1P*G{caKjS0$uTN`%cEw;5uqnS%i@u6HZIsoh0#NI$~ZApeep+ zD!l)?{UiUlIdSGn6%P2`SV?9i7;dOjuf7r{RV#0N<*1`;=?hIK&6Sx?%#vZ#oC+qT zK~Pez$tj~Oy8<^UvwkoATMO#2IYy6<-zuaTvT5Mr)4+~GGGPU{*SigYDX++<_o3-v zFh0r!6aatpz^FBfuNN9*!KXE@oE}^jaAP*A^$5aiy0$NU_v2;()}K3T^r9EV8j@g(_xML|gY=2X&@Jjf z5oZ>^R4jhL(!d&%bQ;${wrsxb?>i3ztJh0*Nqga=ZfzT=F_>uI+xVl=ESFh^4AKiI-q)wDj4`C&JS8%l_~p6QJt-SdV)~M!;$g2wVR7 z94IpPUuovW{O|UUd7ThR;(Uj&b{uQS0^5$msjL_BH1j$pG#SN+#{z`k8LgesbdEh> zkzpSdLCo~%@9P6=8 zZ+8748WQm-YwzbrhpR2BPt{X$az^$M67ek}6m~%>`__?0fLRgm z47FcJJz(N>S>SangPE+$#DQy&>fq<=fN;!{6xDkkkQ0wvG1HGn)V$BmW=p5gaG`sR zSc~R&xFCzp+YP80l=+^}S=+Gv1n!P6AJxSdlJCw6AOG$kw$t|46~CXJ)@oVY@!T2D z&-mH!&U?tJsG;E<-T`8x2;DkSECkI&NrV>CyrXi|)N1n4CQe~5%%0xhqr*gFx5Q!= zO#{tk#!yA$4C_xGnZ~g7aRWVJ{RB~zojcnPl1W#x*>o$6hMWK+pI)p_({uITdc*(0 zUO-0&Di9fL2B8MIj>ldQIRooBR!TC6Vk!04FT2~t`5r|GTO*Jb2w3Qc*lthvn@O$i zP$B_j$9?g|y;=>TVAxtbl zxf5T;B%fk1NBQkF&&m*DnUY;lbi)%-ir1KbOH0M(ZKr}q!2hlnS@5tYtQH|Ze1G4m zCg-d_FD1pYW#UJ)Z}hWiF)gvb$9}y{IhVlJ2G~f98J>Z9NAG>8H=5tmXTS^6k%_=8 zx%)0Jby`|6Z6oMeiNN5W!hoYpJW5nr4$%#Fk&7C_iF>w;4kW`gO8)=s3b=|+cl|TR zenvvo5s*I_4T)|F?4s0rromB?1fWrJKx5Wq!%f|L+K*)$W(hKhb}+rN2n6Oue$*&T zm&EmfWQlWh@E_e~2wGcFKdiH90co-CA}5C;a}jGO9_UTy@VSW(w_hyw6gf1D3JcFW za^3FG2F{oC;Q6J40j1`&Q=bb!;o}!)8dDBa*0}!v4OC zvnOFj0ebOnp7{rR<#>Ds<(XNv_(ZAVId98`w~Mm!blrM* z6|y{w&Sb~Qc_V-zX`3_7co@(WxV>`YHyqBCUE=P!u_FJC7^iWKj4QG3?b}r*!&4`- zhcK@Vfkw!D0VF#IP2N;7;iyP{FB-tmgVG*Q2JPrR@?PHeDSg8piSkwvnIKe9!m+YD zE`Vx2*uFy2e>F}d{H0{mnEq$!t5nWEfQ|c;q6olCGq7Wa0jitbtDoaQWAf%zJJ4FJ zJi|bd|D+)zbH=L!S`$7x>C@N~zD~us$wdgA|ErpJKsgGUy-qxFlmElM-gB3L5IXT! zVe#$~P-@+&n6hh1!%kiy$OqG4SFhWO3eGO7i0c;gjN^~%nghRf;o9OSDe^On{0sDR z5y0-DxNn?fa3{ z>&<+>)uQXQ&^0Y)y;DlqF!?S2_78;Et`Rkmp4stwR*JLb8jIC$b}N}xp2+8!@ARW< z;4600(HCL%+Q}{NA;xOv2UGY{T74St^emDF=!mcZGPTY7-l!cfE6Px#DNp!ApZD5} zwNt8SDaTO8=6n5vh-~EB%5Jx6sU>5uFL<}9`=qKsfzAu=g|k>OICU>at*?A^^6ctO zZiJ_BMR3Lo$)13Eunu@v;6-3B{K{fIY)$*KC0UZ*}&fKp=-03<)*i{*L_ogyA}`m-h!`T+ppd3 z|L)dxmvl|ogD!;M+(#5~5ZC1932||Qjz|fH@nk4yL;u-the6a2`o|26bASGb^w*fb zqP#!ikX(db+)QbP)1uqIakrlAB@!7c*`F>@{!3RJa&a)Qnd(dYOyo6QVPjlX|0cbh z#AI9%`VgBkcx{-AXZe)jKKrpL;y7Ou{}IKkuB=Ts;@pA>~>t zJ6%3Yt?X{*2{maM1!ZNl-Y0FfBI>M`wKz&hIWXEn|q>s^hopf!5c7JkJ8U#s}p;0IllF;UwzRyHmIA(RBXAjHqpx z0ra`en>X#>B{d33=Aj||2>U4Zb(KL5v<0|naO(GB<&VK_Kn@ER1Z@`4_PcQJAJq|K zwn#YQaiv+n9uo%&S-t$Jj&05Dmey{w^O92(iacBQ+4(_+d>rHZ-8y79<4>lhamn$S z-0wdUeq>V$vgm$CmH+GqGc&VM{8T6pCz>}$0KgmWo%E?#Tf@?&@#0M-bD=^VyOz5f zFY7k1Jkal*m6qm36QKXKE`@f~vORKwT+nzi5w^(Rc)S;wA>B*5C?tR;kZ7?3tC#e7 zz49km<4uDN*p31^i1?G?_(2Lrx_DhH(@xX;)5mW5*JJ%#^?(Rur>47}64$o|!*@}7 zw_cJKLF0stuc0hV?lj5h4rmOTVNFn$!1RT&5}x^|9!wgE411uWxGoBer5xS!Os|A& zs$6_pL)rE-$!J_$s_jM@GoJ1Ft{_E0s?1sAj(sqSm3X^qFd?7={aGbB+x$qlPXs~h zo~2!o`;kyz!W&VC8yal13YJ2WzWXt;>?CMHu!okTi%5nr4^A4;1=l_UWFbt%lw=y8 z*INR@+JSx89vq5%UNYLyIv932qb|)QO%@ql0|V$&RbLz&9AE=6xW+iK1N=w=2WRi9 zGjm^e#KCU-0eZt0{nikIFlx)j9GZaQnfu)GGG{uMVu_~ti{93vJlR8xq>{pw9NQ7xwo{c=@rAPYo`v$8xROpgq9E(Ps~fWAZWSjZdRQ74wfE8VHq^{WLwz#`eZ*ig zPMwI7%3*V=mp2?IB3;=|exWZ-xh{=rb1w1XDW&2%#1~7a`!cBn(?@kfJx+H{A~@zF zK8RMXu?9;9yOylDLy`SN-Em3>S1(TiG-XP-=L|>mC(WMwn${V5^B(|@;T9zWGldZ} zt3DdzY=`T?+TboBN*{rVApjt#Z@IDb=r7(y-4^_lkA#9n@&)<`3_ONJX;`#>WiT}9vaMj91g8LFZ z>aA4eM|KM!n&D|-^uCF1%xFv|eLl4;Bf`uOY@1|NAyO&ko>ZhG8-&&tr8OaDAjf8- zt~H~*SAOSKzCUXeu$lT;7GNFGZR2TfY)$#S@cH9sg&`%)#ogc60?kYU^x7SY;qU3q z+mz&}qIbVID8Je7XgqZPSiC){uO%5Ha}(@7YvvcqLM)JH94vk5vzv^|4Bq81$NeQR zP8BpP8@%@QdEVp1gr*(ISEy*-qu#|yu=btl%vxu&kHyN$9rO9=$um}$1-?tN_Y)4}x`=dwAovpO|e8=sNPneQ_P z*i&B1yQv}T<0EqcuLfh#GCrtZ2ZFZc6$2G_Y;iq*R#@={Z*0T8q`|XcPRSwHy8AWD zO3G&=d_p+`@xjl=6-O)QrLJH0)q3uSYj%HkGxgIq-ar!s@Q@Tb*^g&SMlbYfW_Lbx z!F$hZyy@SV89_lXp-QsIjr)BO}l&{%c=VBt(2uZ&2j2_32nr_ z`$`02a92y5 zux^PS@f}DNPK^gO>&NxDy8l6rl1BZ(*GY5PKW(wn;nVE8B=pC0u$z3C&mjIP2|%cg z+yIv9Fa%n(ub~`l>eO4pv@8L|1?iD$h>36@d$6yYp<_$oH>LqpVb(gzNjnv}Fa-oM zZqp8(4(8FXU%hp@zdr!L$}&Ms|9)61PT-O5o%@l)hbDlPBz8!9Zh;ZmskK@ zjMBGbSUY{@|2bGml)r znBF3~I&xa@v;L+=y>-6ztpkc7++xy_>Xe3BR zP-Bz~IAcTO;ah!*md3w;^R3fI{ZY?~cdzj$1Af;thy|3-y*~ni4)&^*KVxi1-1U9t ziaH$)BV@e;l1U!=B_eC2Qcju4I@Db+)Sy~y~02uRv6 z(e2+C7V@B#Dh&-|oJW+uegdAfjG9eiAs?OR%-U_!ZCp)8=0tS7Mqj5Yu(WSbWiGjrX!+k7CI$KT0? z@8j8P2D|>Pz5cI?e$Nbt#Dp+j)FQNLhed;V<{$t>ew;ClxYyDTLE}n<8?+jT`WJ~R zMYg;!bZ(M&M&r@3)PCV?%u=@$5MRA-RF|aiA~K%5f9Hc~O+**IAjrx~QL`m~4v@v( z))Es*dsZ6@XdMv^bppG3_Y^pgOV5!Ge)#~<6metE2ZqW6o*U}7#*I<{9s3Redld)% zm{p^xn(4|F*M~e^8cJg4ZJJl?l$$f#Za<_S|ETkSqOiqB4?C{PZfFuZ_4t1`6&ftS z1tE4I!QA0r6~8*@c%E;)+G8N z*?pd2@>0_BCVC92{7tZ{oQiHkL*q($A!vDl9j~HNG5*;J6C?Ra>`?Zx?_CDvuIc6# zA5>Ne0kQEz*EvLvZU^WM-vRmnh6o$n@}z#PqS79npvWhm2uNjdV|c*$oAN=mw`$F8 zilgcynWNEL3pSl!EOPD-naz-#SO zv?{wDF()>$7e7z_m>2msBw-K`5J0lA<}?6pV40e!l|CQ-m;0Ze_@I4?>w#-`G{Jw* zhG&)H?7E#p5wy$l zLGV^3FvGz^k}OxlC%P4=a&h>;*4SL_d%}_?>X&Y7TvraNsiB@R=EZgIuy><*0Eu->knPij9!QHTq_@nSQEpl>?EQaq zeFaq2UAi}M=ng@;yOHkhl5P+|N$CbD2}L?Y8l{pcEJ=N2s zQGJ(0SBQkqY5+n(&BcB3j~_oI0wqFe*b%SC`^7It7pBoe3J?Y89;-P$mEea`Vr8?k zDCU=%NueOat|ia4CrA-p zCLggtvoPP|m{UA*F#&E@kkNAiG8Ga~cTac4i86jzh4TL^Q1372@skLRLxBD{umZ^e zPf#?O2S>#Y>jm&!B*v8s!v!M^Buy@n5pY}Mf_2;bf#FAPbKCtEn-u`nSaV(D5lNlj z|NchCFEDr)oqj932hO2aW_>4fj*geQbait_V! z`0tYo8(lZr3L9-er=Gf#F(cXwE80~49ubM7MzGK`zm~1#cdYr_w+n*{_XDsd#7Ia; zENAd=aD67g%z?L?h!li?k@)2NHwIK36~Qz*qMOHrGOs*1nF|*%HoD&N+Y_ZU+SaA{ zrpxIXh>{pOIt*Kild3N+cMg29nZ@(mH}X?`oy6PLjIHQ+1eEer512CkG;0J_dt0A* z^w^&u>*6zYoTGkm!SwN)jaxbvrhKNiaFBXL?Fiem*B_FUE_tx|iW4asO8bV7NR}bm zQfo$Qzd!gv)`+S@TD|nm*r?@C_5QJ0oNVlzpD*EpkWW$za$m}cVIiWRyjVV6urYPv z#(O8gy0eE-W%DNLB;$919Kj3qECbbmiNxv|{7q(}kjw#5Okzxz5in14W()M+7I~ zW~3y{j<-D4YB>el+h2JN>JsI@Hb2nITinC5O}Lf{DE07t6Z=f&qJ#n_OnBr_N2OBs zz0OzkkVtk$fpJ~c-6~y3$+o*2qTyR z%tKdlp<(v|LEjY+J`*E<01LBmDX`-hHJmJaANNIxKFp{xpW? z9Q`QH>_ytgRyqeM16MV(O^w7$TfDkS5%sc35A5Iz5mHK5Pc)(5BZ;$0BZo4 z73odv*MZd{A4j`;o*#Lpm#(U$R4w#-P6RKhle?LT({hwtr}9M^88$meL z&6W4x@K6yx^PxWD_UrO;vRI@Lr-)6TRn}m)tiX@dXLnKj#hBIc8Fm9%cAEwooBUs| zUrCVFEc5G=wLr14S#)4c>mL}n2%da{O(5)5&8%DAK1kcFeFdcaN;_*c!6dtCUF8L)a4DH>I+qJ{GmD+<(s+U=6 z<(h-$=H|U%K59;Bsfv#OYPb!^al^)u)PWaiAcvlmlia}4d%Sx;mpr$r{rWJ`b}2$V z!{v7A^2>msMm*I`n=eZ+kzXQJa;3QZX{`y3dngk2Gl;L@!qN;L3CXBHA z$Nh0@R05s7x5n7z(ooQNMMXsy8Yg|>S4PN4(f(IRtMsOG8L%?qM$NF=d4oZqswO3=~adR zo|ggzDw3Y;S}5cZ7`Uj6m#~*CxfK;JG2|jLCqYk&BgHPdlj{7N4>R!w#)|CISI|1b zcSU1bdf%LFx&6#Gjy!y`iWG&YsX5ihTf|ATb3!KSHF{h!=OX9j9==2Nhji?s*b`m>rHn;lyQ5+VDq+}AkJO1R|h$4x~maZI#-qC{O;Oz&W}J=!c= zSF=js4QJHynQ%1^;JVECG=1duxVwV{v2jSLgqh<}!aAa{<9;0bxSLjfmW51KZNH>p zR)V2kjfTqL4h>5RfHccM9Y^nN)F|k^Y@&ivg#FY2xn6j~YEormuaO(HwaVW2{;3w6 z(lKEve=l|r`jjZzZ4L#)*olV4Os*oZOWm*vwguR0+~}MfbIS z{!)1--oR`GSoGxe4)w;|p8ntRzSoPo5~pkLUmKO++8I-X&b()t3dNYU<7f2s&xyo9 z-TJm9e)d5Hiph_cWRW=q-EKMoV{QJEODR11V#>F|S_S8D7qu$WQ5Yec3j9pVUX~%d z)8P%~fUW|mAtwqW4bI=}>+IB0W3fYe?+0aJsKx%*YxEJ(bW8x(U%}KjJ~1&5J#Bt^ za#9XNYH1nk?yO6me)eCXCz)hVHV|zFc<320m+7SO)S{uGoq^)+7umIH69B3$fMtfO zh6C|tMCH%p!`AHUsG0=|U4hx~SFh?YRr#pPXdT5*F)>U>s-A#>u#UStW1e zbwC^&Ah1?M$u(W#RgXUX#%1}=#a2PBTHE*Y_-j&FyEm`QOtE`=_1PTqt4>#G z`SY%>+lkE)h_&=wzh>$7GERciMh@xRnr5phHI)xno=QMb=!4>w=U8J=ZZ!HH=PCcc z*3G8{G_wWEV<*jh6P!!km>u(?_hHS9ep(P0r;T8Fj&8j}me1W`oebxqg)dkEH3gna zEv}pj<^~b=PnQ_D+Jvf#K4{X+ExiFYD!`#Jgms=g1z3W_T`OKT)$4R9T#4)1IABBV zyT3U*oA=Cqn5Um7=tKGryA4S~>Y9oZ_Wp zEY~eH(!Tp=h|OILDSTz!93sv`ER?Z}HTR5wL1o?E!@z7?x&lI5vwEbUmhMYSP`~}c zL)BAPx48T2u4$l_PrggIvSoPyr^}wRf47#k7WwH+2oxRv_JRmOccB;q%owDoX_GNM z*0ydX8bF-;GcmhSe1>l3- zyhUApsR_eNy?T|Um5xTl)9bmDM)$;#q`FS)Y43Q5!H@*_mt3PgcURI(8ssZ2%b{`P zwl!EnNaI2Y=~|amISsl7DV)sAv3X`*qNcn#*mz${X`MjRI| z!M?vp-kzS-4g3E)=F^>T7lJ{oM%DQ}8xj(t;N{gw{;N_{0cfs8G7L;i*s!p$Za3fS z>Lx811qcrg zc3?gH>z|PcFNXOf^OU|)rh_5EWeaH=K;R>K2B1K#v$M0}Pp_d#i;*KmA3a*iemSeD zMQf4DB7|X~Zgpts7+IJbwfZz16|SVuB%GZ&HSFf41al|tgUQ^Af5_!H22ECDkzgX`*1N;EfTEoD%sCWt zpC)Ex6qJ{M?$(r2#7klYOqp;aG_RqqMdb35s2RiozaTHR^ zmwe2u(q4m&5*l2nG36eO4c5>Fu(YP#KEAJ8NX6!zuEUPSa)W5FRt`FUY?_`@ELn?V zoE5`)4^Kz#|FxA6VJ%R$J3F~>vgj@b^l|>%iYf*JA8@!(M|3yj&fd@l>c)qO8XKLP z0h33{iF{T2sy4}0n7+i-<25j-exI!C?Q>A0EIN+}-%25ko z^HXH3IS6pS{V6x;HcE?E^wz?Hpzmd}_dZ>QkNS_!-TrC+^ws#ASN4zmIqMR+Y~k#pF3Q>ffI&v3Wy~ z!-o4luZ*U~)I}0mGv=B;sH&gb@v0k`k(3+MIO=Lh&8iC&oY&SS`G1k>yY4zw+W@0q-Pe3rao{4e3O<5^b$&7hM+WTs3LbWeQCx6t){WMFA@PRR}6U z_DIGLBGkpv&L3+#8Gd+!ojhMgME@89A^R3+`7d`{!rjH|SqZr;qmMe^9qBOr%vM@` zD?wQJl7aGh>klLBbPip%f-AZE;Tl>~sQr)^BSISe{r$g-0ea~)W8~grV`N-c z=XI60c-ER%`>%M%7q;p4ELfu1c84|qnKD#G7fl$Cx~t8H)^AQ5_44gn zFg{T(3D3{t{O=(3VuQYJ%)27c^bH`dx>&VPHJM93HrAF+JNWEX_ortt?$V+UzvHxm zfYhd>8F<5;032$N>wQPg$*Eqa>u+#>y73bw(hdnmk&37LDN~2zc3>_RI-uP zT9a?|L9OU&6S-g7G|Y#g%2(EUTR51)TP95$3JnYYtAh6P;NmGGv_=AYxqkc5J4MrZ zr(yn`Ux{dlgzIQ)WL(Rcy#hYUVtV}Mwj*OHtc)dzgRs(WB_y?n{LJ3yxC4VYw{tE{ z62U=NWu|lK5sd)R9!z8!*RsiS?2F}eYov}EkI%<4JT0zwu*ra;MW}sy99`JRnDUi} zc#$5_?DI!^*&+d3ZOqk=YO|XO+cwDXEbUc!x}z)v)z48AYHE#m1%6-=5;N{1Epeg1 z*UMQWpDnVq7JY!E+!k&Y7znBU{R{2{iZ6j0IV=={eB~5c-Ps7JVIUg%@T}!2n+D83 z=aQG_U>n{9>*J25$;QUUR&mwv_#NyQg)qr1NFq9-$OCwO8-HK1o9P%nC5(gerLb_b z;wS0EGj{LG1CDXiiVu9~m$;NNGDr2w?K^(*Q3Nk!>a3GMitY@oi`YIs$r9IGXiBIA z%wd3kZbHyr8H-J5NDoz|lTo7W^3vw;&KEQUi}8Y^Lc}^+tO6#s#6jsOtFaJ5yIRa8 z-^`rPqFU<+pTDTgyJH4J3@M^x5?=9oYuP0n^e!<-Hr9SdGpkecPxn8gX`(i$(-(W1 z#Dw4DhxG3h|pVHgO%D^0e&bj%h@(rb?7t6Ma<6Mt5z{Ahbbl)ywaxt z*v6tPd=sCST2$QJ+-GC5WPKp!(kjQ(`{z`v89M%_fb49_n~NEfh@*qq3P~^wsK`L^ z`A3xlv4B(X?F5=;gWroMAyw8>j770e(wXk@*O625q#nWks`2P2ZQP*EH=2VIHGb0< zhDG%|rN8)Aj}6HqMa-EALCeqCCbV!*;Ks`$&z$VBlZ|YVBbQcM{ts!b@}8#AqL&Og z7p*upylV7%N!#;huH)Jwbf{kon253ku6V|UmjYg!*rD^M$H>W%{xXV)y-lr-YVa$Z%M$p z_g11J@yTX_M*Jkel!kz%E_I{Fkb0F7y4j#zmgmE9OzHUR_t*{*P>*jmymM>@lbQ|@ z4M8l`+3kMWen0$FI8c}kYUT!6|4d*p6E^g{m;wY#{2w64u=q8a#o8jf?vAOATAp1{ zP+&P0(crwZca))@Y8t46lmsC0XiP&>!@;Da=VflG2{B^inB{<6>d7r8K~^j>-~F_t zuKlVdyG~Kzvy1Xat+|GaXtC}95^P*N))=c6{Yi)-JUxj-_ByraVBakrmq4quh&?s^ z7Z*)FkD$Ci(*g2zX$?&|*?C(i(H3PhHVUEVQT(bH#>8Pbt`5NvxS&d7vHRfWN}-Of zkAktdY3Bblq5I?9jzhUKRbzxk374e(kDc4jYGn zwqk5eTOl;-L4E6*Lzp@V`r9+sy4nb|3-!uF9bschtzpuS`@Ax@sZHh5Mq^92ipS3H z5@$uIXu=+s3{GAJdPAlU>yGv1r*umhXxz3A69du^3Ub_&c;_k4W@l5%?L7cVphf6Hv3E7CuE zGnhod)_fB9lK8<~~2o>F6^>Y%(Up+>J$Y+o?)pD;)!%@Tjfjd6EIEInFXk8(56q{q}mB`GFSEqkB`lw2xEI&vc{>?6lg?O zX|a?2TkHPR$@{~?z`?<>IDy&QwQ}d7AD%>Lu2eic+UK`x_bXw#kUs!_(VKFdv-r{% zO(D4nEIsVwPp$h`U>!~c_Fox7_h!X*uG}DtOg_k6Cit6NSmke@qBEBzX;@$iC{wKg zipfZ9*c?FkR~;MWq7s@8ipF2R8o{yVAT#D`72+*R6I zeuh%1q>~MOfmuzZwsMj3+W8It8`}NHgUtT1JwM~UwU)D`WdZqUWgAjpwIqQl!^7=G zm5{A#TXC_{yVjdG;=&3c7-Xz5g^vS`C}OsP!D~bG-gl4*iJJfKO2lO3<8J=F=&=#u6>T@reJ%sHcpBz!~RFSt&TLA;n8|rqp@fq>raqSS%Td~3TRnK z9$Gr%!^G_X6O(a5bES{$+{5L4z47aCd|~oh*R7q z1G7Tk@s9~}7-j#swrzlm5B?uA{Sl!Fr>rHwzo%NS1Bx~wSMiJfR?DBqyWF=U=`zXc5~-jO4D=t5UTShj zIpe^q2^lRE&VHIN7-Ul5ax!mWWAcnbE!-<=F^KVJa1b-i@*dTNxFZ47LGAvdbnt7weo#e14=KuDsSDw#jAj$Sd zoi?OTV|0A@sX&MC16^A%x(QKQ2fm{H5(q*HzH0j>@VQ*{D_|5|9t$11dH$)x*!(hR zvM|Mmm3h;Ay3?EU9tW-?qiw!ks&%nCCs>fzJ1DT}`@2)4C`LT4eYl}m3gN7j51xHX zaV>5`N}=)`=`TcV-l)G>q9(t6k@tR~H|VcwAraI}*R4Edvc+E^Fkhe+t5->4NAy>X zU;7ae;37OF(!E=199>+5d1#UuAAC#O*>A%$nVs1F_bbV>gcqxHl4XG9P;2IE#Zupn6bM|;i3<17|xbtrlqb@ zTM9O)9lU9xgPQR5<65!5GUbQDwv_Qdjv4YliH0aqgHYgL$Z9QOsh6$1qoZT^=8E5+ zrK7+_5#L)mfTtV8zeED`dh8ctFZri*8z6<8fF+lNer40TzSFVil&Une705GWp$Loq$mKMdtH!iWcuHZTUqnxpvK5*lJl-k(-=F zkigWqzIHaiHe;Uzw9!kQ@2Wh!DW3`}WRXjAKmw6S?5_6qcaV||xOC2E<*8&-3NDAM zcuu?V-!G_x+9<_Ru^Np_I=M!2rf)~yL8>|?ghZFSzQ|#8z(&EpF+`w4>P0YQQaIqt zx6_uN^2hPZ$xXR4T`A&uEy|wrnGBrnzln#O7^QfKXX)*&j^Y`unIhh-B7bkfa~AJ5 zZs+0V(FzpMGtei1tN#yNi8_;sxU}g=AYZ;206lK!#-RAWUHra;I*$aAm9d3?mH~}( z1ANH|U^ZsJGhW(z6ciLYMncV&a?O+OR9h>n@1%6l{SZW4YOqSo2n)6=6f`#KS5{U+ zkg*M5Eczuegma=sracF${eZGS`(0*cal0W`{VdKT22(UJ%@c5t1Mcm@=H&72n23Y? zAmMGJGek*CAQJ^3chLZ%FhEnN#WV04bGn5b)K_kZA-enZNZoIk|GxiNl=HSbo!^{#-`cOSaq4F>nui4D-uUP55t=&N+NjX3KLVcSY!S!p2gm_y~{SQ~RX?Lx88y=6CY zyj)I}P9A?1^jm)SwM9pC4xDe^-#7*Ewwet_T3WgiB#Pz>M0|YDjM%pRi7B=J_)rPR z1+aZmW9sYqJb_>ua^4dOnI;|*&Tlu%;x59{qAM0QO#t>rk6!`pXX{{IR7~5$p*&+aQ-~E7E*>tnB?hGV73jOZgTAYZlSa#6 zj*Kt`H#%<3^%;$N`H|p*hKXKbh<(cxF7#WDIJ+Oml#rE=zq##s97py;Ijx6(&D}nn-WKxa?0IUa%&->>h&8x? zs8DX_jY>iVbSt`B8g^HBJ;0q@rGpqx z02H!!O0r7&X`SE+FgnT4ee89ZRDOH@-RneMnj+v*tldBx-MYBkZMKF^Ha3VTms3y! z+p=Fs=wDd?>#xshez0f1N zAiHqAo4Hfk^*2?XiCpRCN^1tKCAAf?;PB7byYl)Kf5FaT^^M zA>T3^_Pir>JQnIvx2p*O7hO}Jdb1N3`vebK=ns@ohDL9QZl3cl+pP~`Z-kOa7W#G+vIGanIyv=lR42hr{ zVm$0GIy2Mu66|!cLD!s|oL>(gIL$#7-~yQO?$F#ear`}B>OjCP1INioSbiF2vT4^d?TaSCVy zh}nc&Rm85-MaibFF`IX$C>i2JW1{xQz0+MeDhwNUqZYFM9 zX9)SC|gIPRe0YKbsx<9nzgWHbo$_P@KZ zTe_sCW%rvgb;GPvZCZW`F=JUuES@NS%_@9kMo`=h)$n@TuJTtMhK1HPTWzIRDjps- z?RifI6V4r};y!$oRVB=SwC+!sg*_Hn09Q*?DoL>|qacHoP6bT-#T)d~W`gFV%!w3^ zM>jj?+#0$V@fPn!9458E{b=;Ux9aS5ui_ijv;8`l;k$2xZZ@jFuBMn4Cd&BOdR_qEqnn43?6Wz1jNUA!%2GYo}kzL}ZQZ{JHgY{_2%IwcQWqU0ht-grjf| zG;K6}Vln=}kJ}}43(V@$CUdQz)7*a1R;ML^F^>_j-j5BWFATT}Twmm7t&u~B?b-?| zdzpd|(a;mBXSsHXztB>MXEiF8LZ{sgg#BJK))C0*$=gFla7jf*C^Hn%miRjy{dDxJ zG%%V19!S(h6wrz=Z}j+ytqS69&qNm+e@&w!d@|P@!QREiv6~C#S&>zOuUBK7J&EI* z)|@|3^|ffw+&V}oc=w3I1dUVwB-99mnyLGBE-*A7pb(0ud(W6vnq_?E8VnPUEX#sd zZI_;BS}?uN?jgBOwZYXbvGa|Lo}fH~_T$TsJkOvOoKwt#Jigg!A>h(FoZ^xZ*jFhh zL>c_oCk6Y28=8^CM>+M-74hkqC-+JdB6T@F)O4;=c#c0$V|gy>AnA!`aP0h^<6)0R+%xitA+9KXE z$fyy|Lrp%QCmQ+t`ty(F$X^u+P%|2fw#8If&$$b#Q1{XfrYOF=af@vnKf!aQEO)WB z%;&i{G_G_m*<}7BS9pF|_bWpo>|k70J?bk1QV^|NzUiO4+W?zfFT33PL`eg!FD12b zTX0U9zx-WP58fpl4FyF<)3S<-i!T?~8ck1)&SvVdZ0-l7ysF6z+-?JumfV0{KIId_B3|5_R<8N zO6j4tK?@os1`3H!8fv<6Pw(5lbbd4k4pP0cSc_<3$-=V=a-?XnA|N1|W+ZJB-96W9 z>*?T z2d1HPY6dfzOA~3$Kg3NMmPW}hlK*(7OvB0s=4yvuV8$#CF&#E&$lF;3*aZjiIHs`w zdM;1kS16T_;gD~gkzF;cq!LD7BlIzm$ZmOj8d8B8b4K2Y2=iJLD_}jZ^ zmpZv*IM{fh2w6hmqc`PvFQ~a`bNkR@-!zQsMKz6XVl!|uk8j)_YWd$;Jp^1BS2UMT z^&B-TiLdS@ojh;Ut%U0p*Ei4coLLs15K@LA`6LD~n^j2)Xl(W`P|Jb>>)tsHntFmr zv$XA3atnq@3NXu0zuDtl*!hZR2#C#{bmN=!f-a5$$db$X`{C`>x!F_=h6;VP%Zyo` zhR5O&^Xfk_^28|HaLI2-ZiCy{wLs7SvIERJC6yp?%lRx9fbV>^73(OKEle zG4}kN(rOY$Rx*$(oBK+B0^sg0Na2c{O`c^_82Ab$@*%a2K9>a(1_M8wc@W#J*BowM z74e~{akX1#Wly^eVh~MLC*$IfF6HQsE|EWAIaBZp=;>J3>CfiGMIOnAvyQVPa(5d{SLYhvW4DR+WlM=CeZs zIvzQgqomes=UUWBKd?xSHvhp-&9a`MC_T#ZQ)euRO^)Tc!mjW31GMjK7l$z|Wtizjgr`IDv5 zagh@PQnMURLZ)C0NppMhxVrWD2<@RuV;U{*UcP$#melhA!4q_L|LVnLZh0zO;KKfl zgNg2Mc=}&&9b)`<6+!_RIRJ-mNUo=+$LcmD;H{z(Y3Os>eFkg}ef5qT#j0mv5SSz= zTDGXk8HKrYw35g(Imxv6Lt{21$w)iyj3T=IBBmVn4Fvw?BLkLnsQ}|E^(;># zBf62e-*If#>5Ht9uVPytM%})b+Eh+jmP{o5IutN_W23hykk2H-70!le$g;K(1v5|+ z_z5Ol7o}ld#Ef-Q!BaU{r;)gX0W%qfm$yM&odX|mNpMH8_eKt)cB93ZhA*pg=v?AR z<%J>8*=Z4a6;$r|nwEoE4&o`eJ4o$~*dxUCV`OFLi%D@1LmK}TR3GyGmF6N>c2R)I zzJ@v65*|O5OA3PAXMpOPk&`q0#BWa{r;IfOG(nYqWAjB}Ebm$4cxUD5{Yv6ID5vQL;0I0%9$e84 znZElte05HiR|o|PoEC!mxXvpx0|hNREtGH|7u&SaMsfJE`MuqmLQjLzB}9c6UYtp%C{I2dxaM8#^I)PNdb}AQztyvB9!p(&4K zgSD8x+9?ROP*q;H-k{-dwFr#)Wr|=iEU?e~By1*?y9I;Ly*>goM(SPDA&)i|*` zIw%?mz%Av&1C_+LrdOY40u`Dl(OhfPV`{V%+%4pMr7Iwz(sK9hr!0kGa{CM~jqVg} zGhWS`H~BR_T8&xJVV@YmuH2n;92d7Un)Qroqz)ZviqvbUYa7_bnML!kJmK!CczPDXXg&zg!YN7`=g;+lbbyq-_^{Y#a z*eR4aKh3IVPp=+l!MT{H>(;)yx>mMI&Fs5@xx6Mg6buf!bj5J^umAO`h{2R#vMcX8 zthPhzwY_f{`wiehBXz##a5Z4`(QPjXOfEE#BX0p%OzWRV@2`UHr4~G3Kq<6tv+bg# zADpMLaeBt(qG+R5t?W5U8C;l~Hm1Y8R<2|@-U)x{G5y0vK^j%E0g0a)A2F-_C$_bM zw&B#W(xO<^7b5PoMKL@1hoXvQIq8kwne*d#pP^;%;4;I61KRz4Mzh7 zUbp2GYHA&h@kpf1st8(FjN-l(cMi{Cz*pzqk|=odO45pQG}(s6aDo5ZaC98k7lp)6 z@QQAXzG6GTucGJPo6^=?()6-&@Z05n&0B2f(Vtgt%5w$%&Wiu7Tor>^rZ$~N9Rupr zi1%h^n;uWT}87G8Hs#HcE@eb(CA`gcjoYOTW(Y*Db#;WuWy@}Y@hyuE7w zbubNej!uS7zd=eJIpn(6L$wIw*IB)&O^r=C&0W-r*Z_2A?quKwTSB^iyXB^<3?}t$ zA%jluh+tx$Y#Upx)mIoAgaq{AQFGG&S_D@i0bZwfaFFX(1VAaf3w33V2gB&JE14eG zS8abv@au|K(r$Ho+#!^dl=^7nbEBh~24&Z!vfH(|6W&SYz85A8OAXVDA^P8296D7? zQe7W4b$tqdr#eR%iz#0Un#`xBCdJE|^js2CwZwVzlQ_(zE_)c;RBpGC#f`RTcxhaofQ87BS$Jn=F1RBGK`}#lxP)ZP%+ZRCB~jF2+-b+o3PT zX@6+g8pTWL&p`5xjCrE@SK*NYn}@?1cjz#LUf$+j#OWlq6&h6NCN34%=oy$4>7{%S zFeN-Sl9SRById8)dH#BDh`?`1 zu?10O=Z?(r_N}e7B}lHcvB)-;pDliT(~?f}s#j|F{oWSnAHUBW9zed&2%3G9n8MeA zJ(@2lRF3PpAY{T<;xnO?pvq}(GoQGC}+6IF+y;N;w=NdBB+(#sYc&A`IkZ#FD%;hZJ3z(l%8-Er33F`E>f%bG9K0# zHeQy~FkbC?D$5tl&*GV=YjUR~Y;-!kcT59bAtq>^!}!<{bwUSlcJD<%CmD-JttAFXSEa6Pl$p0E}*%EtcQPEs`<~o;9x3H^#rO}aSM!^As z6ts16*Lg>D#!!y}UkBo}BG zUf#>r0~4*c-kwfpS4>@xaLCXiHUUmF{OEqP{yU#Xl0{7@E?J@x3w9z#49ShOBD+nH zPWaDfm^SK7z}z?PPZRY`#d=hrS{WS1h%AE zs{Mp%neXYUko(rt_)dUg_|G5ai|MzMCbm@IjcLGhOW5q z@^NZ=rN7VZx*)QfhkP%CFVd~UbC}Wbt+1t|71C!-_sBw3B+Em1{@#9wQm@vzL4;qJ zyrhk^ORquV2akM;9Qq!vxgOlrLn>6dV|~B1t!;eUaHED9WioAbf>Zn8QD^sTF;;$G z`!KA9!LS=6$>4{f#{b&A5MzUl7n6{X5wH`jLg#}oyy#310dpth38tHw_s27!3+8cx zPPU)KeUYL_yF^|7DZOTQrYys8kmLy~v|i_6X4ShjI??GfYS8^-S#@tWf01M?nY%-v zqiCrVSn8nmU^}7WIa^N{)Z*Vns(tKqDsBYdpBu-TSl`RXP{J735_;8N;Y-__`tq%u zxvoa^yDv-@&}~%ycdYdq<%V91-6HOGnMXH|Ru2#|#CUSXrDM|y$-h@te-#$c%1?x1 zfzeZQ0yr2&P#BqB6SLhoYtlr<=>LId^2U({Dh4nN~ zC!q}A^ZPk-5UZbBW$SHO{K56m+YQ8U)u}*f6Dpv|TJRnAnI?!Mg0Z=mqgZ`c_6^$o zMU8*c8$~CjN@f5z@1~6p(DXPAY3S=0y0d|)@~tRCt!lKBW$E@fg>HrYweMEGtK7=d zGFujumEr5Y8D}fYFvNAEB1c+%XaycKGNSvTb8kj+wigIn2#{;iT?G69qhIWhNpEvG zxiEGW-Oz>uVoM@S|0dr-97!;%4uUeqa2V?sj zPLl{MN9=dwFh}oe9KSQqHBZu5_%q>K_>?(v`uz~1TCK8PNf1c-^6!?3e=O>M{hVil z&L{|bGJ@$fdsGZ2adfh%NQR)s=$QCwm9|{!mqt2;lcU@)K`Y60HCP29WUFt*Es6yCzCpOfmjJIRK(;_ z`q)KyLg*T75z3k-)cQ^1x%=NWpncewdIhFFuzLhPHh;fto);7Rt`y-bH`9SlgvDer zWnp~C4~J;i`*bH=pr~DOjN?w26Peu>V}4f+xHF*<>cs6 zQKZ7ZO!S~PY9+D!r6oWsHkr8T?Yj61>ARP2Dk{FwsZ;%Z`OGtPsAhw3e2!ZD*TWtc z+ZGo8^?MCIJpAN8l7OD*g4tZ5Q)YbolQZp(dh{id=GI$Jqcy0+T)`Oy%`YM(&x)q% zvpn>^)GRX&86fOeasojIp3?MxQpNvhL`GSWuBr1Ro2WZg;>3tB>E~K|;+OqN6G8cV zldu-Dh1e_UEKp_JE7aK#Sz9PxYWKLX2eUH$rzYwj$|%5p7Pweq=zR1&z=RZ{ROa?b zb&K_(I~UW>PjmxV^m8#ShVp!o1{w!A zEXJkkMGEHW6hZ;%{}<-ac7GNQ^pGAFx@x*=a^#qZrku^S)Fy>aSNy`My{x{}pbk}c zwETkm=6=vI@%_Eh z-YBm{j#ET?I5YoWk2y))6X5pb;y7wPSNUF4w4d~W-x}LN3@UC-4W5tbJI)*VnZEO! zxLQgPK>mN+7u2t8^?6BHI6$I37L$PPzfnBCx=BIT_q3ThsKrf*o9pJ>-*gRFbeATa zb<9WQlxAHVS4AN3C`i5zg192@MQhWb+e^Lqr?&JzKtM#@C&Xe^VK>p>9g?mG@pB}P zaQ_#l#6abovqSJl)$H!`=tJ~3J6-uMjOR=*zlhS`iFm6pR@TLgx6jIQ(#y zCysuZ&fk6N$r-&4_D6A{Xa8f3&TVZqj?(M8l5>7urHu^*&1_m$SUCQwtE&sqD5a+C z?uTY>Cm3T%#zc6+;n_p*wii&Q=0Ir^8ylOgTG(*@6TzN~sanV~A94cn(mZ1{N#~dL z5W!`LPtP}@p%`=Y61f;Gu(Pzy{@t|7+GlX_^@-z{kl~syDKr0R9}8OVrXAQ*2w%y_ z{H#QbDZe&7VWxxsI&}7Iq0nBvjSqf<-8ReFX`|}q4#~jeF5ZG`G83&%0(t@D69VE9 z+S7@6LQX!fd2Up_A$>iq1Vf&g&OHw9AAsUBxSek*-LT84{oUm*DwK9az<+i}I;k6K z67n@lu~G;HbAWfDzfJjrkcdqH$;Sh$UJ>;>`OHq7b}6@tJV-fR8rPMGQC=z=aO&D} zAT3ua(e3&y8h+g6m#XtBm|!Nf1tJA^P-XF%n_JBO$ZAqceqmjk%H>gr^Essu5eYfQ zb7Lf-Bblbt0N@>ZFEwimh)$`Z^Kl*>wcetLDE*($nbK412cm5)q*31`7C4kkjK4x( z_S$FBYw@aCIt5dv=Y(glRA$3>FcW9>yoMUIv#T@5hKswja2Cdv5c)IWFchZ8zvP%3 zC>}jSa8m%H>~^C#Mx5T%j?%(X{o0)aOAaN-N11X{t-pE{xCFd*ITc?AzQ4XxiuU}O%CttY_Krh)hMIy=yj1i?;Ku8m_iU@D ztav^koQ9DE=4yGp_wTC7f7LDePvyNSfpZp^42fz!73&)+-*IsaI+&$XO8P-?^AFQ8 zCUKIgS(hrbiI&M)?}E5!B(Dg*$gh`g80mZdKi1wnp6Ygc17^2nijX0*l8`ZD<}H;> zQN}W)B=e9lvrVXEC}TofhR7`QEJ9??kXf1MGK;+H+j-A9zjMyxdH;EzPoF;W81}xu z_r315u63>J8jGi#54^$d_C*UjRbQ>17&X7Qe??4B*NHmy$k8g>1J;}<+`U-2JBds^ z9+isE$YKY{FWzlul2thB!orrD^-%5UxmQ`PLL#a45+)t<6to=C(xrdd)rZ@-^TD6$ zsw+82zNor-bYSLM6MLTh#KpR%#H@cn8+I#Bn74E*dM3{VZa5~RU6m)|^f|bWeNlF2 z_YanO=XuFNoZfz}>cxJ21#p}D-T9_|l_+~@m!TCec?^dJkNmW7vCrzwk}ke)*E6&f zHLU6SHj*mh8%?97;8W+ZFS0-B+D5vEGRJypm_J5+EXXykS-&=yTHtU9KnFnvrhJ>T zy42!CKBXh5EmA@-0oh+t5`U!Vh1lHfYP;aj(@-jy)?W63ROhNj#K7pcFG{4I@!l;T zyMM(8P%+Ub)9Z>Te`+r>vTM@c-_Vg6FP-bWpJA63GpC>)9nfP(#l^%ndOvHIiIU;e zf#V0%PEYwA`hk}^roYGOjIZt{qG9upIB z;VJyR#Eh|GvBjgW&SHHzYqy$xYZ{nvtdtrn&j)TG#Xr084ok-L&CgaLq8_iH_y~fb zzE-}H;Wt}uP~1CAMd>QTP<%;K?JkrYlo}BeK$Pm@W&SppzLJ2j8;D8B#M|057|A}H z>+G`y1U#a*u*pBcM7`+re%29krr*RuT|Yn0k)LNLMhoIs)_!wK_7m_G5DJ>6L;Lyr zhm47B3@y?zn{Zxbt0h{M5vt|q3i{rU)Jy#_K?$U6s93t|j%Pf>tbOSEv{H}QW4i3Y zt9<^%D}>bqQ%o&LbM=+D9r*xF-_nYhahR79#_RgIDSv?>9h(fTG zsmE`{TF&26{H5b4dO>4fUJwZaNujA7M3@Og1YU%Pvn3@Z@ki{yY&ZvS^c~BXNW*<= zD8}y<)))|P0@e7RLFd0esMSXrqyzaTLDEozmw%`$_-}R%SK#u_@0Jl*G?Aq>ERckyTUuho;04^MB#Yhm3AiVpsCiFR++6~6P+=b zwEOGA;a>%XFDoTPYTqc@Jo`Lqb1ab7`6AA$y?{-qVrluV`a^i!IThvMzwFWe`$olX zDgpjc5A--w7;ZWmRahwEHkTT6Z@d-@<01e3SpMg6p@J~sHx+JzRl9aCDII^xhq|Xk z$NxXRi!F~R_Wqp4jT?8vpN56SAe7+?)8xcxDl7>8eJwBz*3!{YR#5O~2^Zn!R&w8% z=P2EhDrYNh740`jHi7w#Ai!$q#GViP9PA^I&B6>nkj=}0?PxQ-c80R06Xchba#3F8 ziv_LH8SUQ->=lAMX23}AhM^(f_3PKW-BOgaG zJohG;{{<-X5oZm8SNRe7a85{{{|gha=)4?qFnCyX z(7&gk-xIB=F4H^r#7DdDtBtmH6fY>b>QcajL)zthc*3nUB!O|?TDIw|pSL3+Mw8%YMc05@NJ?H_=R|nH`}YK_s&`+* zEdEc>W0&BqKQ>MAa3Q;7y>~BtA2IB7%0%h&LHa${|G7`XtQ%yR^xFb|&6F?XL_B>; zQn3SVsk7pv50gQ!R*~i5IDRgV%>#A6I*_(n+B_vLu8Z^Cfw%3vA}Z6az? zjX+Oe*B7|0-QQo|Tp9vN84%99iuCrM(+*urkR%?y3;IHaH}v&kLj32`VYm$WexApo z_X6+?p46S|=@#m_z1!#=j$xy$6{cljVY#aCP48;00P(n?s*F|M#Z=g~v5I(!(yPB3 z&Dx|Jq4LJyb;LqoT474{LV^cwxFzd$jnUSL-=8>NH^2V9{&w?nP!$J0Da!uDdNX_2 zNfN^CEEo(v;CGy9FOsq#5SIk`goMhqb}3rX2mSqM!RKM6#1wk*Pznt!eSlGobbY#k zw$8-{(61J+dnAoMPk+BX{wuFy&#AND5LYFI2?~08?t?9#1I`Fe_2L^`Y<*~60>q}Q zY8>wQY<))D*l0M6E_5hQ-~~VB$tk@kh@Xkpp+M&8ZEf&57^s^bm;C@JAcI$|lJ~j^7)nxa-ij6fF7C51V%O*~b=`UU;9OA@cI4P*8PX80F|8%nQ_Mf|(6SFlj zEqtH!3?4ub)?-`P3>7w{eD-e#_3gfQQk3lMu2*Z-iFx&lRRM$u8NEYAd!jBo`^85R zhwX--ZZ!fyv2)|0>O{r`tP$JFc`Y@fbsJi8)}h6DxV&cvzDlkhw1%ZVkpxEugEF&0 z1X_=qDLj-;uk^O$O68h;Iw>{9sJ&*;L((zNwUBo@VOIDYL8%|`Ie#cFe96!xAUVka zr|D(yer`8kNo>Q%eRBEFM4l^}hwX4lT7=>X+AK6#?*bM_Ds*pe%s*q@3w0*GFt~Yf z9A=fo%LlXd#YZ?opN|_8MwEtnev?Y~dpmLlFGkPam7vb`2H@MnSsAA1xtL%eA>>4M znT_8!B_$Y}TDo;9U7F}~)+D5>znKEt+Yj%f-etK5 z-!6?D$m)C&j{(&MH84fyh2{fQ17R)6Cdjn4q{bMXdz;{LE0|`ft@Zm!*?r2zlg@Ry za#>P;^egOHML@@BU!qZ%iILGNlcYwup2Um8ctH(2be3aW+VTsRvw6|wW8o_`2cuV| zrXn@1y|_g`970zUk~@*-=(ihjNvyt$C=>t;(Cyjlzndsa*ZO_jkCEUsVV*L zF2L-wGhIQjN<;UJq8@u6XOY&UIL6ps?C+A8{cKzMlNVOaIrM(|xiwy`04= zDC||dX!mopv{^f=Ll*&1Tb0Fc-MXr`;J#)@v$7Q}`U3OotnW!L`J|ZOU7@whv0~ znPeL1r`}xxS03tk}AeZ>qx?~lhj;K;zqQaPEAzdvh+gV zw(K%8MqY9I%h#`8uXslWF|rjNPd#6;f9|Cj7%sIMuukx?YHtZZl=J7uh}S!fgoJ}>3C<W)@wOtWH6FYG@7d z+2$zYkJaDOyN5?EPW*(?ZN`rin_LnSUoY5JAIT(8!A|)L`&n6W=iRFFn#jSpr8In% z73m-hCRJ1>R9SfJ=ZBJbxQbimS1xMfJ1mtOrxJ9Y2*Ntsb8(!h3)KT=`R((lk zeeHsq??X~$iuXs-p29TM>F95&tm7Ye&Yo@Ly6?vP?9%dIr>=O!=rj15e9XMM^n_<* z$lobY);^LtJ4DB;ha=S0@egWcMDyt22R}dl`)B@X<_C1VobKB6iu!tjBCb}+EjU?p zTycDOHCSS#sK1sy7u-v`FOTPnb_jy&^7DM<6!twfY2q4UUoM)(JYK>|a? zYA|b#XQ3C1QTh^x!IyHJRt2#**LCa1!uTb+AiL%ge98!IU~}`S1C8m+>6YJ+x?bXb zY)s(qareU`VwX(cm4@l9U}3LXnSOo8R)W@JM3Ll)N9&4*loJ14()=eLy~=}1yCu2Z zd-W?Ea1=h;%^PkVbj02!N~oTIa+tLWWDAAh=8+jFAUV3IN@qM`+}cuGu(JKyms z@Bt~3wDmTA$(nx}uEkeo4|X1YNx$AW@72KA+|uGuMlMWRd&3wslCGRs=JI*=fmVtr zZSQ-S90T{`3@qN=(8*`Q?Sciq2XNMSOLeV@+vPwoj_+E=HWS~q3or4+^*~|s$}%FX zuw3O^EdzE&S31vXH-qNxPq5R{r0I$xZ^S>e4nxKHet%iu%OXoSLy96YB4yp)yma|Z ziRl+Vsku2EW;csx&z-f+FJM}J_{S}MI@aXYq($QRvkq@A zxk!X;UhcB=j@-tUZ;k!P+1yu6^m*E=aP6aXF#!Pq?wDx+)ibSQ2Xs*-QloJLy3Xze zA5Zd)5K0IenGOD$j8I>%Z9bRF-miR)HB!kwETEK$LQiIVw8`s{6;a(jiZmM1=W8yD z14(ah{w5?vsiSjBfPWsr3JK3E+E2+7`VwR1^%Kjan|LHV$3`G=CEB1 zs<~Q{PR|yG(~>XcnW~=^7%E)L76~2iBH|A@j{ZR-m~@Pw0VHznpgkHT8Z1h8=QF1u z8H$-E!HX}U<`QLkGS%X4W^nrEwBkrMH+H<6@BuqgwZ&x7Ro;=AFdxo|4=QNA5ht7{ zJ+mkqiHgpN8jovsxa?!KQTvgqG|}hx4$6wS8Fc72jt@UL)$9lq4@_4wdbv!C!al`K z;Q>lqFRwu!^MDAYaNFgp>5Iffer|5=@+p7QV@J7E>}`^8z88iFRkn=Vas!Mg57_^6 z^Zsj76J13)oMEP5iqGI>tHU~^Lq82m?DtD$|bys<~Ci7@0>s`}hy>#)fZkTC{)cp06A6K7G_L^>VAk zO#Js7+Aype0sd~cd5?G`qx3}Cj)va2;JX1T?_98k`P%!@TM@m)bmX{W4PGA>xg;MRGd`~yaLlGNZNVzcxUxgvFhVpEzV8peD@FCn6)pzfZ}Wx>o@H~%P@P=jRz;(WP22O1MZ zK%T{0OVrQ&LlWmH38)o8&T~M+U^$8w^PcUGXE{IWB7&NN5RH@mQ< zt3OWorIi^AwL8#=+Llj8yl!h<9q=JC{zk>>Q2^ulzM7Kh`5&UF&ZeEuC#~D}-d??< zq@tpEbMHP(oQyG$p!e$hWTi;VmpXo25X=wN-Z3Ws)ZV`N*clwjSY9h=qfFyC1zrJc z7cpd8p>~ii`N}RST*J?!acGOoFX_7n9jk}m!!yxiH881=7t?!gr?m?uZl+KD1tl#PpESn{t~gtI5np5MMtm5g~ufI@K#)Z2G; zCpsByOW)QtWWNBd6vZXL^u5(_WiRR?dVky}*D2G!$Ovhj>j3rHhV$o_JMKz+MGSDm z$}KY@<;Ts;PL>U_@sf;Ble`wS>bnX!@b!-tZxik>{<0)|I&Rt;P2ZMuyj|~Q-MNN> zmGR4`@O{{QF9w8+!HkBQrAA-}xum=%VUZRX~D@s)K>r z3qecQh(KM?$3@(FboQn&@pdSlcLxC4Qlq1`pMceL^XAK8hgLqFJ~D`+4uGDJqULRf zORw9O&z*hSC}J}x!p>@Yp;d|RI~tDKB@~F3yhdnH{AJV^eHTA}R!!DLQUtoKQ9QbXl2f`CNTQd-G=NNFO-`2f%Bk=Wa;eOGe)(NTrkyKiHF*~- z9TeaWyHof85H5!Af>eAi6!yD-i3rPPd&#}=HaqJDYWRjD%!!sq$A&(A`ZS-{y97)} z%lIZyJmJH^L^k7h-4itj`?=byeV-yY7qEIOrQgZxNbc$Ckhm`Nskv`1S{OtjngPFn z4tkp+JtKp1xQFPe7l1@_4;Uz@%gfmA6B|4|b#)cSAdKS!3no603W$X6+YpjaauBiX zfy+#>9!CJpHjI!I%5sNR{+4|i9~;Zn#N_ylA==m=ha_hPZnBWOZ{r=F;*4A;q0Eo6 z$|<`7U7!>D6>0cGjFTR*Lxb z*KHM~>e7@n7R@$g3IhD=gGq@{w0GV6f&$dRy!|8Z%f>*lL_k9e7RK7H6e(&lxUp-V zF^ae6Huh8sYY^Q{-3QBUv#%Adx!w#BozdsH!-)u5$i# zgA21bf4_d_1R~CaoUE)xb5ZLcOb|L$a-OS4;rmn#*ckA0K0#5GV{)b0=kak;6&tRS zCXL5pwq2(`zZ5fVp$6G>ttkNEOQ6#2$}Z@+CY5@tmv87!5VL_Tb(MyCa4|?nE9F)9 zX*-NvG6j~O3q+kEsBb6y$Rw|M^*I>~zNKQYnoR(XZ&_Qfl`$Yzwzm{;9OqhSl05Ow zW$Gw{#%>DxI6JE_9)zV@a!HV6+z#OC{- ze`(aBo)a*CAl7%T%TWw#UpZ%>xNr(XvMuwi(?J#E)8Wqbz%EVMBWSP7-qbxsKhi_q zwf(K8$Z%NWUVn1-i}uT=H&frQfGWzD^~Z0xW^tY*Bsi|Jl0ZIXPS$|m9oMO5RB6aG zQ!5TOu4;+wz3`)0;v)&1eSYnF=cPJ986m?j5e=-n*XnPDFpBf{R6rs6UymOCiDja@ z>h;T+lOZ?Q%$t~ybkB|1E$7`k60ugT1o;3}HeXB~NqWDpJAJx2*Qt$3+&v?V4A-6z zh+X=;F#D^YXM2X?W@jDi*~`()Wsc%q9^+&3nWhX+ANj%3r0}dp<$8WRO3pN^a{$&c zrE1;NP76=mWyn+}<=%|yo?XzbKEc7DdtHEj!#;^Ed;ztab)5pk=8pB;GYU2ekYNkI ziGSeO=&FC;FoO(x3FSFTl_7W>YmVOQ&=)Dvw2DO5d++>M??c09w<=t0-i9hvG5!7# z=7Vs$K$&D%6DL#)>6A~&gYzPiW|mhluLTa?>*(klB|;Hh z;=-*dwzIRN(=5?(1nXYYJ|#s_JT!&FCuTk01e|H2z80976wLx5CFZl}r+J%po$em{ zpXMA2smSJ8qPzg9xuKt(O?_DYW^<~Mu6)qz{Y5id4nu>V9+4@dPgDPAlfkzIa8cY; z529qhG5`u>sP_PoeFT@nla;m|4Io>(2*?7N7`@QF$DkT(RA&Y89uMn;b|M@AiH(dn{`;k;Ld& zkR(Eam8MJ2&L*oKTCaZ&d$Nnjo`pVyr|5L})mphQ>xFWiH#RTfEYFe7mOu6hxisb& zxkvSw_pSfQNFwd;p0Ay{mC`PCo}2$+58bRmkc+b#G6|EhhQ!~t`nMYhoGpS6QW@XP zf(8)YG27VKs1x!5MW38JK-G5z+^+m@{Qc*d#4=Gn7MaDF!~-YW&}V-m$Mpli#D4XC zc}kFiIIZ3~f!OMJG_)P7+6JRIb3#^6dvo%LzuFJ~@8`mKP|wM8Cs`+#D^_g5q$iBo zz83#?mWxangJ_E_^p_B1ez1s{hyB0e*8lk@rl0uIXiuFSLUw#2US4R;U$am@HV`%L3ks^ z0CI{Gut7nbeY?in_G%il%RpUY2;+=hqwxQR)qlOb$x=)VA+zW44TAxjl9A#1*G+%> z)w!6MNvG!9EVn}cGfe;M=OG(tj)cPLs_xT`e?FG~xi29erDFO5b(O!Kqdg7ZFroos zL(RLJOADa}61H4KY!nH;fo|58mO@_3fBuDmJbYu8@)t0A`Tm59nb3H}pRT@`e%O0o zMcWq%u;6@KJ@j+C!xyK(QsXAVjE^Q!!O)o~D=U*M;4Uzr9khIFjY6x>i)4Or|)OIDkJMif`Y5cPrgij?*Imki1;s!@NltvV^O*9e&B7Z$Q}a&p*zEv<_&1ieJU zJM{~MV*-k|z0631O!Yh64OlD&e6POsmpvNP3^6f(4&0Z%zFTT)A(@qL-uOwM>k`!D zLGi11L7GzVW?H6JDVte)Z6s_e)RXKFxK~v@9Mubk(r7Y~fvr>obs@>hJcuWq9>E|T&*MA1WSW%Z)g)&)cT_+z(s{lfv#{( z?T+)|kLu@joThvbLjB-)ktlALsM0ulgzHlia3xk)lCOjmAh8g!kYE(P|Ez|jW8~_1 z^{?gkh(@(i*u%CsQKgvtf1SYpeu->8XjsPqU{RT1 zeK*a%LX*c72}h(Ctw$@r-u@^9gz(n1CK_FXJMbGzNim>z#sXeW)FmteBr3Q!Z$3cA z5W9{;w%o>;a(C3t|2VP#cdLfKN6`~YZY#bWuY;>__&W$w6(gL2WoVvyY<@M%&dB(5 zyY1b(cf+21j56*{f_pF0xC1!tp*ViU@~H(epMe`rtvGh)|Nhkf_;qfl!@F<_+p(aN zZlTEHmg13RR5eJJo6JI?63t7Tdmp40^PmoFl!0FU9EyU$I|EGLY-No8+)n(s+Ux_L z+hHuv_qPmYpE0qbEQr_Vdt0ZH&++qLv+~lph=&a&p8fUtz;;28%sK$#5d?{sp;QZO z;q-9eg}-0rBP!&ekPW*R8DCuJVCNRI*Ucj9SqfFy5R2FP^@{z?p(Qxb*J69XQQatS z#dAig7{*ryIbsa%BEG8z$Da&;Pri&CnrZYp$1NS5X%+*bws;=VQ{h)JoO!#Il;bKX z_0nh|PGB8Td6xVzIq#VBk#YRGph@NYu55pZxCA zcOmR^lWVxxOoXow%O^9w*Q26?abc*5wW}eq3PwSD?sOH>vz?ut#>ywNG4&9EDC18S z-A9fVv(MfSsJqeqb(+ctfwAfU*i%{|a8I^TmE@^_Wijc=aD~N3C$8mz z21Ng32F`nT&mMN9@lX=n6c(MQMkXRa`g;OJrpW9ZNp@0@jWfxd`tWZeN}+9j_~*O8 zSmQ2|;gO0h=GBvD=zqF#c<7^OcEdg+VD6@uRjTJ;hWP37?Fr`~aC@3ol_U;kR@UbP zg`YQ7unnitLTtE>SRa;*6$xc;;(H>qf>6xr7g{oi!h<2@<+fq(b#{>3GZB#k8(s4K@mv|tJD6IYxiqXT!6C$4w zfKRM9USB?yT!V*@=X#QPl%1WObr%D@kYIM$>5W6JSE!zN+Iwi~0atDj=8A`8$|#M> zpC!FPEFvm;)5a!S|MCbCy_IYfZ7aoWysxa)?q`d54F?V9_7~ZdS2>Wa4;i&B4Vhxy z!^(_8y+BM8!G*BJQfB?rokj?K)ymr1Y0TnS(HA3`K1DV!h*&)(EgwatXq+wpk!Uee z^V+AD2S_!{^UL%t5(2eQyZL(B+EecY1Ozx|DQHACPLC5qrA?Yda)H8WTpb0`Ze3!n zW~fx~M1rs=d#&3UpCd$02^dIVq{6<0-1RJRPLT4L_eQ}jH}ibAt+9<~XG6R*16L0= z9FeybRjmGI)36EQJ4jx-qVn;OrGzzh62SK(A6eL+Hb4C5JXVFg@m`_Uv%{as78ToU zXVs`(DDC0NO*5M4|CbkoQ@kgtc^=swm+Nv znQEgc%!7QZJN8zJU9215L7ct@9vpAR-=|q1yN`4HwcKt;*V2B(Ei>akMVyZ1|M8?+9{;t9`wBfkX4LidR6O1V*oX zC_5l9+`+zkcP=wG*VUwLYp#d0zDSc91@9LQYFfuDWoW|fZxKEnjUQTZ&$)q4s| zY~{kV1QQI-apUwJVK@oaJ+cc5x_i5GZ_B}JnETrMO1ScdZt#1#!{aiE7Hm0ebv_gA zVPODb4j-m7Df8o3*2BMq;2eAjW=xNq^zdz&LknBC_uJy}!@mSY!#1mdv*QhdeCd!U ziqR#daGJGzu=!Uw-9LgE21~L&`}qMdY&xR(4F_MI`&P`y&7BW)XRtc{ZVe-9I@hmL zmc-K`J3EP#nuexKVf{7f{zC~_bOCqm4DYMm#R@6n)td{~R>Yd(`0%~jx3f%DUhlhz z;~%Uwn&nhftufh&p||5r6=Z~eEy6_ux>g0qMI+3oq|u6qTp{^y!ozP*fPPeiwh}~^ zg^LNHdM}LC_)IWSDEeWb$TLwOuaOYR#-Wiehr0%BzFYW)krUEFw@fS*Yw2ct-iFD2 zf>QVrmxhkceRqwYnrv@_bmM^gi}>KU%q9iJp_tUv)OIpdGY{Y0vqw=9%ucKO{kga8 z5a`+q3W^foH28?YwmUTbhby2G7>q2SYgz(2@iOc|nzwJ;L7G+k;QJN0p|n3pg5#JA zlC&5;oCqL}5k3L$vDp=CAI9Iv58bth&6xtggbci0&Gk>gV)CT8MwonZ#+S7#`%T_e zeUWnfy$;vPnWpCp02n#!Gb@i&Lp55xS(KZaQ{}HhbKiH zn9P1AeP?7Z3|k+j1x7pkrDwAYYFhli(wIUuwh2BuL(rbFS)c8)m{EO~lT!=^key%$ zZvoBf!nvNeu9qDhU09)Ry5y{dw2OfuGb)2P(P`R$2mer~iXH;LvAt<=mp0M(L zRj)*z2BVH;JCY$R41Nkq@(agUqz4+qSXr&d3^^SN-YP7JCxeK~=0>j1NcBdxUD@S% zqcv#04T7ZgS9fir8C)$$;~67l)b40LgVIhAG5nd*dT0sg6yWP{QVz;X=D}V74*Qap zNNkcTi6n{kasK2E-}!mQaX$h1Ei?Q5$qSor(iF!+eMp#Niy;t+?1oVYn~nM2CEK&| zK__pvAJ3hvd9U359?Ie!0UPMz)QOxpbR~%HT?DjwP``N0+au;udDRfjw_UdrP88Kb zKEF%>GW>n!Z%)#^OU7UkR|GerOK6nSV&_;RRK+ z&{-U-k&JtTw?koxhkGM~{AD7iBlrnY4lG~EM$O63bHe@i`AG@(3Ve3NK5=QKI79R8 zEdY;rJ=r~lb~n*nFM6+EU0a)e*B2*hWeqhJcK;Fa2|tTJAofLy&X2J#UQlg~e^5PD znarfXPdfYUnXDap+hND>jZFX>#cx}lN8cw-QZ$>w_3F4Yl4KKb2C~{yRj-&-_(>-} zxTV-8eg#_H&^-p2$>M$@l6slV&W{{Ap&BZgRT&7m3O!gaJ3B{Q@;0p>Fc0t z`(Dj$3M{ipmS8%8(D&C-&p}C0(3S;R4D3b9&Vzt+2C>|cu94rH zweb@Q=hv=W3o`HuGS^Ka`U79^M4EiNK+${|w2sQQb2`&5(yVr{f2!9_UlDAI;z5fV@fRBxI z`mDShE2-3c<~3i>d$Upb22 z(^gY%lun(m+&g5p27rnZo!ef>cvu3$cAOwh{dKik=!DGX0Q>Q<6sx%53`h(u?*j76 zP22?Ks#y@TOYBlXD}SS&+jR>~fy1IFMhJVujw{u%U7{(cY?`6uX`oJ=h{cV9-irM_ z>!8rKs7 zs8=b34u^=b7wI~~l}RKQfq)<2ot_{m`{8`THxrY?HZ%4jjk%>I3Hg%d>)ptOA5f95 z8nc`jB5`{6w!XOcJR!1rxMWm|p{9}1vRJ>pBgEau*Kkk}^An#OxsxoMb{~BPw<3H{ zK}L!EA74OOtTX7LF;wNgQ2=>Iptc2By%k!B7tpWIE;+ZCeCqvTvJ2IH7KGgkmTC@u z)5{Ki$PCTA6|j})a0xQwx2Rnz9v+_7p1vMy^iR-ZE)=O@P_?3L*2F^4=s}`b7YP8& z*#brE6tv?<^Yf1OK;403Y;c%k_|zD{Cm#fn^OEa0!41boe{!(z-UK{x5E(Rs;H_h5 zXjA?w5ALqwtx=DPtKmj~CI34D|K~?=M#!?ie9^FbMcYyG!iYxfCr)^FF0%1eR6b6@ z%c}}$__)A0?7(1=)rsX@#D-{NuBYkRg$eX8a3NYwJCW1;H;t77F5Kv&SF#UV8LTo# zl7A+1=v0q9;w|Qqb4hrPs|RS^8u+oBKqM)4`=VcBTLc`nHiTnQ0hYoB-$KO1?-C$S zhmpwKKx2bk^E%zd@+B94@dSm>asQ~o2k6|5?=RUHH;rH*p=n>&K=hX96d7Ur2npaS z1Yt?3IHfNrD8pJdw)UGU) z<8OvYMcI9Qy<>j87HwzW@jx4r6f@A`O;8wo$x!}Zkj7(!p{fhdnzJ(95?5EmJX^aZ_Q=#Y8=k%KMF-OYhNcC{XC5}|{BrQBfoLoZ+2VsHacK98r`#C+tRn!nHh0DVW%@o{oWp5* z>Tj;b+0(G)3!QgRnYqJ-Z7iAnIeGKWnZ++Vp3x%xBScE^MufOCEKF0?wavSISvULM zn{^of0Fh@(RpGx`6fuGu8|GJwcwgDHM|ruPo6Q4);P?e_Z8kP(Oc9UP21e!*G!W{F z^-GG+eE<6erXw|fm`X-^TdBN1gw#m6{29T0c-LCGx`csXG<`g+=Ot0@Z5R%1Kbz%$ z_+v^zMe&Ti-{o6~SWdRwxRLpFHJ|%}{8t?WfCSTYTq9_k7)vHRS=|i@37ND0APElw zBTs5xi6N7DtGIp8>?dnZPLAazhB~9XBQBBqPZRMEGBUe)u?_wu8^+7O{H4Pazz7@} z7Ha=?_94N$=MeyL>m`B%mgVrzt=}7%-FVb|WvG1c4g0T!uIC{GN0F$@wuah$GY??% zG*8+UG{wLb3}-G9e2s95VJvBcNMoL69E!)=mWph=INTBB6Lig+Zj*ro0wr$YEenFy zx$Hv*xl%c`d=w);3}<2wEgFj{)AweNWDXU&yxPe&)56xCE)+qBq!m*%_&E**epAEapN1GY zS_pB&cOqkAB2*>E#xw?B_n>y4>p^X+d`3-ZF5WxS+#r#3))Dv~=a0H1*1Qyvdy4=r zb0tBIG_k%Y@?GzqhijfCGy|_%Cnh6({m6g2rFg%0*8VWPao|>G+2<2k{IPF})OxMk zHEX3b!@{nIS?9A}XA;0gP?ml#qv^U!#wYwsimC*2_=r`L!%(D!}Ntt(%o%=Js}>#f1Qx?3+@x zzYpCcl4mpq-`3dp{4B9)$kJ?ksv@bSC?5@Ml#`FmrCRmXf*@L`8h1Ba@Y^@`MfHu5Q2cGV+uBu`Qxw4gsPUu=0` zExUjGc^Wm^FskCTWjw^7tCLI{Lkfq3fEnn$;0*>_i=Q_SOn6voy2L8i4Bn{~R9iPM z2MsE8U2gTMyqBd~oznEYYXvpG=bECrIC&^0I zr80NTm4X&pCj)6r-VP3!OzT;Eto38KmQh40AJlic5@DYdaDju(V$BqE7I=&O4B0id zQ5+oklXrhRPOzR=e2j5H{~;}zz9m?s!7iV|1@rti%MK5`Kg+VhqWIDd^?N7zAl`22 zZ57dEFvhHUh+)CXaOEh6`Lcw4>5~2QmQAm)>p)x#%cs!^I+`JuUq0=Tu5X%O>2Oc3 zSo@KG{iSJw)V!E@U;fWpT3QWl-w6PxVn^^>^K)rB5S30qaVjc7>{I`(w?0gaX zI4~RLQ~<@CMl&$ZSFy=Qky491R5&Htr}fe!XDzL&UpsDwWppoYz)4|kH)K@OcBIp# z@5@OM+s<~v!F2zQ!smpE_r03bOH7X&nr%eA@|!X|f4$EqJw#?xdg1#3OO%7`i39)s48R+y(8Y88CDDOLP`CuA@<^TN)izo09XN+Z~CPEP;xsiyaSEsj> zizgOm^Uw~Ie&^4tK5>6w^1Y9BE9>tF8>62q=`QrTE4~T$2WRWsi(e$>Gmepzxhibe zzNdB41Awf8UFyiuV|n19Yya!zp?smMr{T5C8t&<8;2LF0(5Q=8RLStl^>AJd3ewXe zZZao+W_6aOaxd0PWM#7%#iD#)X8!%{LV4j$yDiaG z&jPi+zGrA=gxdvL2R?pYTz7>*`@~rFld-({x7p;dVm|%iN7IEepvS6O@uQJ3B)@&A z_%0cscXIg7`BvtSFK=xeFdg(yjd@%RUQ_&od^@#&cv!i!R^8RTcf$P3af6klZ(-uE zeH7(`DVve@jN^gKn}7gltK6HYj4AUxqc*yV32DPe)x= z4yIL-ahO(Rm1eoFId*h9J@gh>*}DD2=H@1AjDh=rk)qzm0q)|-_Do^f;TvCl@9^h! zS#>@hcQt!UE_7Uc*{M^WQyhWKY#`>>O#szoh+JYaeB3$hEHd8Z0kbVju_mFPw4%Ns zSN-HO-`{c@{`vF0+B!PX!ZyjLh^XT*kP7tn_gCak|5~5ZG!JU}!4@w3f1q|!$(u&) zxSnuZZg@u}ox+YOS&7_L-mLDOXJa)id zAv?zE;hQ)n>6XtT8R;osNJ-W$iK*AMjB7n2ch~M>QEPpfC3ks#x*yY8XT8T!Rf+Q} z3DwudKWZ~m#sAZGaM7H5%+}9hJGm%VwyWzLm94n>%*`}ke`QztYZ;=K-ujaRd+e@3 z?Hyz^5SRhQqw0-Dw;^A3`Ep(H0ik{4r3|9Ks&BE&95~WnLzAa9I&l)m<-=pxn6BFI zkls}|R))Li_{Kzy&XW5a{{tC91HkEs_E-q=$9)$}#|e%;t%@heXCWkES+^x8-|pe7 zI<9qFGPSt44tQ(j_yR)EcY)gh@Co>pgoJ;>uBC zf}54WyT3mkl(G=m61~yD;dY8@mW#1|dL906UR=8^+pctf4ZPk4> z2HBy70D1Kw@>y@tujR)Zw=4g{TU+8o$d7F=g*oT$Kec3j&_G!XuFoayFi3#uJe5I9 z`j@w8a}FjsMv;w);<4JVCr|4Yj#(IGZDCp|F7ee<@2pimtNAK}OW$0cdTI4ZG0&#M z*Z4w`j=Y~9CdoMrC6L~kEb=azR;!BSDAxt!ko@7em(rzrg5Q}Jr0!ffo7t9GE=lt) zPRiBRU~@^p`$sUd>p}4dk@^q|(V`%*FBAq`^+FdtRLXp~PH4JO99VuUco4|5Jcc4DzNxe8mPv0?3Ta z+`suAAt&-9R~}pgtByecYoq5zH2%68EV zy0!(`t#lCG5*!w1E7ByGL#fOk!U&GO&;#|CLLA++F_Do2prW`4lWH!%ZdQA`rTj{E z_=^LBUFI2I4h$uw(z@B3My8`%)ccfSAuFx-B*W)4Q6-SHlTaR@rbI7z-i5$tVy;H7; zn|MVenUdLCIjYF&buYw7oWEufh=srJ-Rl4F<^D9Qp_=Y;fnKi@uz*W) z9^4loTD|$*4iB3HO`bj=MsjQs>?H->az9bHevGI?MS5xLqWQ9y1|6df`&L-_J|FM7 zIm$6cwqfB~wO>^P>%+szwFD%$0$E82>7G7S85|=jMye&Q>LF@XEj;@k54*zrt>}*m zThhbDQsdMyZ|{AKC>hule{FFy_O7Kr-M~q=X1a1y^;hIuPI@)oUQ8CyJ^gM}vT=8D zAio$!bxH<-={I<5gzq@#tI(`8(zB_c#Y>?dJpcyW^RHVY}Oi2K4z(aS^Q*1gMyU$%ni zpTemkCn}`m>ZG{TR!_q{i;;(h1gLMQhN;pK2#Hl6*wH;S&s=7i+PD7v{C)OVLxM@B z`|DTvioZ)XnB_)vq`mza&+%z9Z|IBZO|^XNGy6iGp~KT`%KV%4>~O2R&{Bx>2+;nP zj*_h;SId^4FBu#W6#I8Q_OFl1v6y7nAp6#j2iuAGKd>k{3XLTKG7;wG2c7UUbZ^93 z!rte?z7c z?79D~DX5Ro(I5Lw|gL^TrI|2eT;!Is_A6$#^XHOV!P&93*ZLrntGlf5w)e9P-dC7(&pu225r>EvKu zp6hfcqL|V{c$Ms?Br^|Yp^2LE>kV=!*%(o~sP*83%hNT}^Z=i6)|&A-Mgzg_96;q6 zPC#n8Pq1BS+9&*~NYst4VJ{J1`O2btg&7CeW&|zI!*TVBnSq5Xb_s#g<5<6=?{YO% zC@9R^MJZUR&F2g$NYRf_1V&+g1C@F!DDkC@D_7sH-c(V1;!H7HiJ^WiYH0R#uTFpY zp#2gbFQedaBeO?hRM^1i!lPaXud`j{wu$}877h5E`X5UziWigQ^!EEghO%yzSY&H^ zuVbU^#Jyjc21qB@%LiRgzEbw%rj_MgCFfBO0cYJ8?We^=MD#Aq3O@MO(5~|dI7_~m zYsBUQ`s)Z*tiJ{lA00a?kS`$}X!Q~M^=b~*TJL^?6 zQ99k`ueL5zbwGd@zgl&ujk4jnYAlz%1lT)MR{TZw*(_5GEFPFNK z34f8)pnx!e!f%ryr9xBP>f7l1G2ha-^%UR{N=)+!+6h)tZC4M3CX@+eMDp?-gPmZZ z>tEK`uG0%Tvod-O+~jA=n3~%=_?Fw_x+gO9V=>eEE4xo&8!?rZoxv)Ve9)!YC)0Bw zeE|nBwvU6i%xg5C11?$2t+jRZd_d?JqoF!#U&F zw2l1qIw8bP-MUk6} z_^Ds%G_1O%i+Kwt>zp(KYEk?smVEidWZYBf0dS6TC=i-Gn8F*>S~jp`SVdUd0hy5lQLSxH31^Q{H$Z8 z`&HTsP1Xzv4n27jFREu6u(a$zZnMf|NwIn(KR3;VIuQ`pj_t=bS8sw$5^(m}j*gB( zsFI2#^5XxSDav!eDcGkI25md6tEVJl6l{EK@S3FBQK?@sFvme(&ARqlUOfV>^0g?zlNC zokIyTaoyKA;C|Dhqi6+bBYjiFh%q{g%svLi&09els<4;y^ z19z;l*aMCemYpSD{|R8SFU=-~froVy%r8`tUvU5GArsX4XOnS))|0R?tNq0dp^)4E zYTE$<-d;H8^C6Vi@=UM71iZxX*CUdPYRY(4S;^23;WO6-2x4zn@3ZZcGgIdjK=))8 zBqf#l1Alb?4!#b7!{y$*6n56eoLyK%t3PWWg1T#a{ zP8kvu6@Rj~f{KRiv+SV{&^7x*N+QO&W_bN3-0*YSd>5yiH2itCv_EG4b`e9j@n>&o zPGzw}LMxW^Z~c9A13j@=+;c`L{ zj@WN(+%lreKYQQ7b;va5earfN=C9UX-iO?G?vmhO{!{SZ*nZD9Ur>l&_3|{#ZR1gQ zosw26SQ)>beuG~RW+mSHt1l%$er^+RhWK&m&RsyjX%b91EYr`XRyO8>NvpuWj8-@l zl*2B(ffG7S47{L%(5)$Z-BbbFPxP)mk5N5VH~y8dWLV&F_or`Zn$>o?JGSwG!=x1b2_28Urga!tYyfX|>*(3R3c77B>VWax|i}h+o z?DBkAP(4m>Aol2zSvIFOZ`5A{861u4)*|7dWiq~v+APa^swC3a=b$BrHDouu*coSn zy#`Qx{p)BW)oek>89PGp)j^OZ!K5Z48N?<-h-qJgDr0Ol^@om*&IBg=0Id8@(H4%E z;KZ?OI&R~AdLq9k_EO-VZGZ7aK3Uma$kQvm7QV)6L=YVF!98!NRkdgmAE&(`&s-sd? z$myydL@zzD?$eO`2v@&Se#(RSuI6yc?~e2>H-7u`{Pq1+cnBj%b9sOmHA9GbK~IUg+goHswtX`k5e*5QdD$lNUA7FAL!*dQW)Qz z5XQd}c@(Hi*$9lKCl?G>{Hsjiz-T@RHW$4C$&-TaHTe0>uF-+CCm<)S1)twbzpB^K z)~>M98k7BJ77}=q(9ZY{tzlvaAb4qW^cklxQc#nsEQN_P7t4LCbGzDQ;X$$aG5I%_3t<2eoWx%I=Fn%@&$V|s(& zI_^Wrb;k#_Jpw$5HoGBsyTR8`H}GE7-cU4Vz;>KmVzs8iv^B+6fkI6qKAx@Az@YNq z5Bh`!lOJ|81tvN>0H5+8^=J6UB!wd67Zgn1oz27KW%yytxPf+r+Qpa6HbTV{IRv!z zGo&cL6o|)T4~qZ6w%WdXWghFE~<-JZ6-5tF8pZ z6|#XZp>Wn1XC){2AZ=P6q?K4ekdj7Y?+>Lx8R+DF|IDhXPx;Qz_Cr8<&5uV+d2?WQ zzyotx7z7HtZILUSeL?xIY80~Jk(XJ^E3Pw|FHmmxZpeiHPBdL zVWEko-|4=yP>EH}YnwH~z>Z)N6KAr&!7~m@EE61^K~gzF?1|g*Bi#`D0)w**9slwn zZ(@d|O*RkDE5hBiy`P`!>Der#afZmj{O@Y|)7TA@Qq4H1hD3+J9643!E;)xp;H)bX z4^YLV%z1%302i{;oTBUZ&TtC%YI!~gd>7~QZ(p@s^S_e`nxt7DrixUfE!?{tg9M8U zRu7R`z$jGCJ(n(b`xTQ|UHJw*xN{m&3u3#QRFF_#pLXki9V{mIOPyPk6ppe3WJ>&^ zl>WJ$C3A3+>zOjpcjvqb-UeYKkGFC0DenVyd>#~83_r#k4NUD-h~1NH4Esj2UcNQN zBMeHz0?lJAkZ8*0-zWRfRRGBla=nCyCvR!WNICewrz5bQjA`{FwIfC3KN3CM7Pnx$ zs5IG~k##yhh@_Q3`2N95;N>)c=XVm3*Z5u^)%|7}H)=Oyt>SJvejKed<*Edigam7Q z5xi<3N?s=tl2>0OzBNKlx#M%|tqXCDBxH|FnjrZ4ygSu>j;%g_Z#t3x8NWzI;gh*_ z#0Ukw{`Zhq!tO$SEYS9=AF&biwX?hkbz!&T;`qe;5f6JEIkOr6tTIAwypxkK_p4h^ zU0eAhT=Gi$>4Sf69B>C1VO-2(juLpl1CJ?EUIkj5(0#YHO??T-a-tD$aLB;e-Q-ua z$*LOv4|QKSJj1Ia<9T9JGp$2MzY^I^m24)+%VlydE}lO@IK2=olhq^MOCfd+5^O9n zqabV;83ck=En4wkA^UGlC20e^@cDlF;s5+ktH}rG`zvX6jSiBR>=}-pmk8lNLIsob zv9Q<&^v!4!q0P8AjiZ8e3BoT-Z}cjBFt5c2+LEU<<%t+!&souJi$5WF;U=r3M-0{k z_@IL6G9oiAo|D{gf1nqblQPaZzoH7cOm$NKTj&9K<=bB}4Z@^u4C8kOCO0W?^d>d-K{w$`a z8=CSwm)v@P7rzFl?eZOr7k|!d$MGiXBZs7MZ|h8_@8-Pefr%?{ridf|=b%1(pgyoj zEH|P*)k@4dQs?OJsGsra+V`qJ>=JgV=%FBs1^B;DC{txKh<$Gc;xXXsP6F1IJpf#Q z!2PcD90*W;!({P_%Yey26%AbgrW|FI7+A)82nbq6Aliy5uCJd0EgA+ml@5Hu-2mo2 z>VPw?T@8qIK42hi2Y_QOhkRCj6=4+YH%jWf7r-4;hD&rCr*56*>V7|{<=F%2gLC)#S84~HErw^t_4M+n!aFS`( zgW-~PPc!s`7CwPTbbv`v4T43iF)|g=jD9(%W?L&DC^I&;pP}IK~K4E zP1W*rz$Jr&?qEuuoOQ;FRXGD9VN3`UfTHQNO51pW&S<-bfg1>k0G!DNp&f^ZowmOUt73%J; zQ{A#0$jHxV30Hg~5rRaXeo9C%c&uPN3@hPlcolcV!qV6-dA!h+p7&G(4!3-tu`?h| zC`@U?$uo+3wJ42Wi9;&YfM3smJIMx$CH7~B>qAE>dQ^VtC$sh@m1>8rajUN6`Q*Zogn$NIEEhBb@P`GX?@plY3;<5sA^9W>-t53u)>JQ?wr?Yh3=M6h zLrL7zADev9Q^Tb)fsdbpB_x$4u!BMe(@Y>+fU+>&T0^yG_>V7e?ne_nt;l@|d_q-3 z!mC$siSv2V!w9qUOtBQ`8q#=lUDLq>ldS|9v}joqoUcp+41r*PX}G!k$su!d!Xw4K zVcu-4{p=@3msV=wxNl6-k|!bkPrSA`Kk6kvpk9nv?7Uix+;N zblR1K6s<@OE%Eq+%2p}B*b*q^vY?``Z<^I-g0teQZ?Li)hElk62g&?2b!ukiaxh@P zxsx1BY;yO0$G(r|0xt0Z92|5N;5vPdR@%gWi(3PEP}IVY{TK`P7RK3sB!yMW*Y}i# zHF9f=6p!wCB#q_l@lT-*%y7Yl%KB+`JA+RT_)yFyz{_Rqha7-`NgpbnbM5L<+Xv$s z!hn^#1l`-dgOdLjG0Rs!jPnju(v!v0Tfj%O^$UzEOtee@t2ZN(Q36CL+(x@_=#h%a z7ylfOWPy_XEi9YA6B)`>B1|LUhx-NR7MEH%a#l()LWgG^!KzR!ylJf1eriJz+nBIT z%T*V0>t9T~ypTtmr0k2hSl?R-1O!WdFXIwKRhkr4L*f$gYt@xv66Kn4ai(Npyy^pV zIQTr5p&^Vz12V83ritBqym$Yvb2D)39(*ofwtF7%+5K9u@nqs{A{8hcH30o^O$1?hS{?eq%Qs*gb?ts!c(C$ z&-8g|Dbjs+!%V!E5*)?>GrQ8jG%XwuM!uS;4F2(3Q!*>x23kxpH3o>zeg;r0Q^mqa z8_OUP6V_hXKgZ&GNqDv%V~W#)o;?CO=)a`pUzrnfUhiYv66f4*@}eO$%9&klT3~J< z)*tSL&n!VU$ND&mn3a+U`-d8PpAwt_7d$YE^*?@~ZVD|Dvo3~^*{qq&9SOcipm`nF zH0DT^w6%LukKdhEe|P?0Ym5vpJYp91Y_nqcyz^~84ffioo{>ZHu(~qhI>|gSm5e&m z?-;pP4Z8>D!|9j&^C~uq;tX4m`SGm9kqI8cr*^+eTLh$suYmhfcqBV3O1KM`Cf@e~ zldQiwy1Gm!W)!cNc9*RM&I%6qe2h03gxP*B?ls{H^pH<`paRDu^+wygrU%6>#)A?4 ztDRLFfwM*jlT==;PplKK+mEzZRo%>Aq}Lje{TmV6h}acztN6Fp4-Bvm!s~4yI>-4L zmdGtnM*FQIYqN7=6EniZy|nis_`%LGBd89a!g<(0RACIkQ-tAjaKh8?l!DEL1pai( z?J)z2I4(w#E7-6h4=`pwYshpz;RW@aO~>seD15Oh;L_i??A5zBPidB}4>$>)5`&S& zOAOO~+GJ;KW+(f~%GTDr!T-P?(PyvCU-5C>AGH2<;94ra1leKKbVjhD(Kdv{U5 z7TsF-+RpH7=uhFQK0l6n!H{s-9)OJ{IJqC#?KB(#A3W14lf;R&XQnSF*w>zoQ5TA9 zG1r?tz)jJU2(KO$k}J-Fi+0OVt@Q!txO6cUtZZpgM9SxHt#oXbheL-vNQcys&PXWe z@&?j3y1hf3x6eL#GNn;RJJd(4==}+`%F_7iy2ljwn8*6KCL6rLlvc7Ouz)yS-(*ym z+o7O3`4!Q@z4ye@i+FhCc;x(3>JO>|$9FucnRx|wLhuNWNd8VDZ-b5oVu}s^CM%Bz z+MRN!H7qEXgG5nmy@*PPvu)Qz~ny zz7yH59|lpgbSD8^1AgN{K|mCNQF%V<=|ZJ;H6i@p#3+ps9s3RLjOL5Bdz(5SVEofjdspWz3=Hl9JuXGGiR? z=~ZI25%_!v4Q;tc&)`-~2P=yQ^=U9D%JAoc@Jz|ztncSkd}@h;B!j>phlmEW@XInY zgH=hB|7TY8{`p)^ih~QaIjZ0SU^A-0!i8!e0f|!IR4)l|i`XUe-S(MPvONIFhZHfN zs)pl*2!(OI=nuf^QWGFOltBK#6VxHbQZMb(3#@0ozqoX9Zr!W#Mq6Zxf-@B&0KPP{ zN2zvFCpR(Mn!B9gnGE-kdlEWcfVdRYwoa8f!+4@uj9lj$N6TzB0NMotgcedAkl6B< zd3YcXSM{c&z?k+`SQ<$xEG7S6Szzkcj+*GH+DxmrqfpByv12?G@)d+fU?61Mwcdx) zi@nxJ??FXniI^qvXiCG(_1>0X&_5}@n9?qVV7o#BY9s?(C^=pXeEIQeO(KT!Ts*J$W62g!Yr8vDct2IakE#d^lhvcOsbkZU~nW#DX9~f2v@GC1u!AOH$(T0Dy(A)x4^lPn+o_%NqNHN?x2;s!%Fqf zZSj#78+N39aPPC{Bx}{_$(j8@)7HTdt_73>4*CU??U3-tTJ;QfbPng^gixo|R|&M% zZz`l3uDg}_rx0Btmft0|tA%<~d%B-&Ythub{(tCM)}hUS%e?_;{_fPTDeN(bJZcV4 z3}yv2ds)?elp}KJ{>Q8(xGLO#u4YCUH_OeeXZ(4`>)V3a2t-Y%sV*hkApJKL0A^Xi=ZS<& zoa|EK@xb+kBla@hU#fxb$qB6!h`~Yzn9s!*!-;`G(Tm+0@1I9W1K(32sTy^JQ+P*c z^uRk0MT!1S(^jlGI^h^PZQpb2%2Lc*Pk*W4$UZ-D2kFteuASoMZ4e_QqyETPy)X@n zSayc`_xm1iKNfKB{-*$7;kcMdLQiwGDrUfb1S?K*Hz2R-xj}-%KwM#aIbVi?z$3pxu zx&ydZDfMa8gr@hoX^zaqMBP4cMT3tj*GHrtBsyBQM!wyfW=@Ikz_@}1mO5*|ADTGc zq(1`#3Om3x=fU4^!DFnEK}`l?-?F&^i@dgTf>W1w-bSgkGUVq`tK@`A5aK1`3oq_4 zVyj61eXHSQxaV;b;e#TRQC1fnpCr5+?2V6I66p(Els||b-=r-^x9~t)+j+(93e#%& zUOM0+P(ysp+ZqGj`WefTlCI>xYnVg8bR#+O&2f%}EHsq^3$EJdK2Gred`8IJW-vdZ zp>`X1x9JwX%UhwdpCS!KpUF1Ror|1Ycp=g&RfQp;PS{q&dhuLybKBBDM1bJos+`Y$ zxD?*>(pAI#%g9VuMtcpwAvwNy>8B&P8`Mq5@79EpV|mcGcoZ9pOtBRMQhAraL+cdQ zGiplwjP*!4u@rjj*Rc;1He?i|7v^gB_~)2eOpLCzA7b6NA@PKqwk3Y zrAa@Zs4?%ao{?_X&r1B^)g4NNSZSWGx>#wP$EB76N7`EEg1*qCJgw}~|2eUk2JFE= zmD!34f8d)LSfnB1KZGGWLKFRrpc`n1b6|uK|HU)$18iz?0Bz#U+uT@AMzRgeZssE( z&>L4=kP*rs)nqo*kcW8N3!Co3%aF4Q(KiH}xn^;+oZw=z4fpxuM*#bru{vp2bM+#> zFyg{gX9l0RJ-azPT`}AAVve~TdOM$`PREp_3A658*T^VE zi=Ya?h_*@oO?gt*1Z=$+>_p9(kd%v?_J@8YHMSkK)s=A>nGE%QO6engC!d2+s_75k zy$R|B3`gdFf8#Km+Yj{TT<})=T!|kpu1LsN6icXuQ>32}eBH83y0^vNfdEhB98gkC?SZcRMtc!MlU?_ZZsE+dUigNx9?*ZFV)e zl&{_Zu0_*aU#&_+CAjys4If}0b7WZn5B|}{#4tg{=yW+PpYv)eWnRhD#({8Nm8mK3 zTlz-NcMp%d+jQ%kRoHDKXMI;qt3}lLaKIurnuZ9HF-(;5E7O6X@3;RlQMfl) zmR8Ath3;7NNq8OSHy<}3NluZ2Na0D*(%9J$QUHlM$CEe?e7DD0Vah&XYf^6yjs&8k zHOk6XUK&-7>7g;?{0v3G9=F+vB2uRvp{Jumb%bo-{Vw)=Xj=D7yC2to7SyhlDfPaT zAo>wqDAkIW3UYWn1XWRvqXjxuuX}0EehM|thw!ZTl&U+6bj=a5-f03Qn@Oai{H58| zf!W<6vBrY2_Su-DdyPMP8Z&dHUyJ1vgTrgJG@$VjSO_*fzd1n`Xn_r+lB$ynA^Yj3 zmC2uNdnV%V*32%l5lPj&=rRWxVR!SGFsxP=K*^jpC*sthE|TUvqGi-VlL*&M^@|x>~b$151qp|c*?V9 zfwJacZ>m@c$*t;3KPNhbP1E~*(yf)TMdiPf??zWj@_%FO%*;PH2!&kEXc^0{C`SyU%Z`*n4lg(?0sTS*FNWAyTk* z4{Q3*0nj^Y8rJ%X0W{HLFpbDZ=%rp#j?cL9%R(o&|6U5fegRRge5N@#Q5i%n2}Hc4LA$p zh~9IVF+&5d$MNz=MCSb=Le=9|DNL{N?5>-BxkZ==C9ZP3|2-Xp>f$9KbyC9Em8>{Exiq*Q;aOQZC0J^Aa`XBVy19w7fkh~#T=S-zfE zgY8nBfmd#ws}5LC?ItVUywR!mJ#k4~RAnDUHWwe+{&;_@3grKdjHZ7MPqYAW(fElZ zqVOfj=>a;i@AX{+If)wy#}0sdQ0!B)Wvcb65iCv|KFQ|W2-?Cw197x%WH@?DWTI+? zr{)74e!~A0hXAya;eGD1oMbf##&FwB%EwdQd6{py-JOLljz>O~1 zn1a^N1B(f;(+kX645Xd{#2XB+0}Cp}h}X_KlqbzD;*hC8&D>4Y@Ny0EYN zy$5y}+{K}DOVl^Q!b`Q7b!sK{0E$o&%$SuMsFD#%Oi*BZ2$sw<$kj792+IL$SWMFfx2@amgS!a)1sI{hQz zj+#Ui(bv50#!Ck6kqhy0*iEW%zV<*cT0AFa0=6_{b0N{&Q&l3;hr{j|bS& zKR=Ug;+*8d*lUdn-8@f#L_up;Shw%aQ|LOUZ3!NuD$hR)+J(RN*M_GBuKRmt=cdCZ z50>R?HE@Vp74i;HDwp=D+wV%;k5|INlE+R(pVT|lM9H8$LcQjh+Q8)8b?YO6@Zyw_ z6qv-o>j+o7C$^>XBh-DvJjR>e?nl>^75@~hKg&dqs-Mhf#?$Ho14yuhK=-N2%yfAD zd;(1a!>yDV+h=#xYMt?JJAi@**wHs95L-AyzX8kD4VrYdK;2Y2-x|t}kfIKEA1Vj{ zX4`D#7HqFm%f2tqE-k);LrDk)a{5-Fj??_UG@k?z@yNA|8Ej1J(RFpiFTRc`m1jh` zhk>U((FAs~w0za9G=|Sw3!3K3*BGI~h?6?0P9m@l)7Ct<%4$t|2SLj)|U@*>k9q{A0%v# znZanwPJOWUr4c05|NJ+jeU)_U+Un~gx$6%IZB{_vn2<7TWOdx3ESotx@s)xl>~jBW$O68)lk zA85bxr`q#XM%lSD)Pwno-q%>s!fMBY%qd(LT&{_j(t==xPrd>H<9!=CBL>(pq&p~ss;I2@EKqYJNpyb1j^&E~sBTQywURJAF-S8I<=Qs8GUBh7vCaGXpH zboL$c%@B76K)l;3cAkF509!3~O?@#7c4}>r-1EN@UK@R9q_~2M0g2)zRJb!6hYh_??YL{*+vDcpACNyJmb=T(GJ5p&!~9kYf8DLyXA{%?6%iDCwyjk4^pOUjz#^sAHf8b*#u zp46=s!;){7fRLF}=5mqA6nMEsPBmSb%~%SMLZyCS(BVLJ;m7sTtvoY(5{cdf;2ZY_ zmU#8kieXiyD&f8VB(lJu+v?jxV3Es{O>^z&slsAp7ox8GOBU(=X~j@SOql#MVZlfi>~(eTKv zyvq-Vtr+_qj25?pB`LPk#G=SY#lBD^=Q7aUD>&yX4)Ym)3RjP}+29Jrn@Ne;^J%zA ze{{gs8ayd?c%&#G`qBF;4WMHWF)fGvH;}X$uC0XG=CudI*jJ&2K~jW$c`NQUUw~_c z6MY59*ms~T@m20k!O{sBh19tS z|Nd{22ZeFPCL>uRKzAFyu`SBI8F40jf~wy)vCUI;%u5zG&-n%Nn|#Eh*+c%j3{~eb z5?-AJs2S5hu*K4sd5PfVP%yd~J94}rPUmV-hkXxppZuwdxms>kwN-=Dhr+kE09F27 zJFQA?)EL_O3O$B!g=VZeuvW?*cvP(71;g&=BaojHVwx!x3L@4hw`Etzxy@+&SW*}{TNs;p~= z{>EKt@L=B7|9DMqLRhJC=op7V;s-Dd-Z!bCYd6TKSS`nTABBZulMw2@-5yh`Ftu9Q z{MIkN1??h*?OfLikdzHeJit!`Q;psck>9xz7jGVkjGi<)+|*Rt!L$Y~WU8>nI(X1~ zc8_WI@fYt|r?2JI-vDMKrCPnEFs*GY`a)B{H(gtFy_ZBt#{m0jJ;8 z@vX&I5bEU_c;vJ$j!89_kH+@R*1$J$noP5st5DlAv{grCGZuFI&!_2!{D+ElVTaN7 zYr+1d2vgn<+FxPaH?)G#;#usuh7Ys4zqcObEoe)RaMZx#CIP#)4YTni@$&;w)x4^4 zg(g_(D)u*?om{Q`RtWSQ!&w`+Hp>RP=8=Ym7p;N1jVA3sZ9lkk+LuQXnY}Ub zn?eu4fiwq0b%Jk`La=ksrlc(|pc&ZT!1><5GW1Mvy5Z}6r}g!c%=0@q^mMpa z|F&0>fkc2(%{EUgLN8`#QZueDO%pCBbob5iz0@o{ZS5MOapcZcx)9nx=;)Yr;skxv zcqBHQ#V0FP0H$FUN9tl>eTnWY@sk^gG6WG9mY3h6#!K|5vkye1gvvScGJX#e^9k#X z*M7z3N#X&R!OZSDIKvG>U+ZIzixBHVj;j!cF1`%&F=FG>$gEj~Q?Xpv9f03LHh)Qz;&ye&0nZb1~6dQ)L%$Dsn-I7sI2Yss#S>04uXi) zP@ld!=8E?1YKYXB5?{~podj|B9OG)lJ7KSxa^DOPWK%@iT+ttuKMwV8Lql%ShDuBj zR2#$)A9+KxKcw0yF&?Lm+8d^n_7M}Amc`(u#yA<|SgpS`fgT;pLL=F`+2QWr#csH@ ze~2y37YbH&a1Sbm2O!xBS1m38B>>tPmGI!HhQ>9Zbsh)OY#P}Qw0Wm4+WN06BDQ|g zBvu@x?jg|Hi4ze#$c{)pG_RHkk^_;EJu+?ljO@J5m>=P141nzRB*Ba0iR%-LczWnO zl$S93i6UXHF%1_|$;^B;M{)NMl({#ygL!Ilhf}`mq`mV8OqBVkHZ_;rmmj|?5E=<7 zOt&$5uV%cc`iWG@U3K7JnQ+J*a$2xMai5gf0BD(kP2Ka=v`(g<*(0RLe<@xF&E`+TCpOk zi?YS%5Us<5s5imBmc_VdbNQLhMmn3Aa{ug#TGOr|Z4={Vm(zoNcS%dSl{-8__+h^Y>{D z^Yi4Ax|`tVWS6pg+9HQ)HK!kvEkx2TSd9MsBCeXBrmmilX?foj{mHEc`csgKEh*#A zQ-RxsJga9*J$(K9?A2al(WKKV2N+j>>PL23VFjoR=f_f~4C?jxA9M;BMufGt_%!xTSn-*FBm3P-OEJ& z=ZKJPul4-g4G<=+0?h>XFFiT-8CV@FQ;v+s8l^jt9<|wYM#*lkBW!qyUp;YqOcp|a z(6bXku88jYX3XQQW>MJx9-GkHY{dNMmFThk_-xjt!=;4J3KoN$`^9F~YlAkGtp@X{ zN1ttiiNr?U-WJBeXO-l*!a0^=ik~ARmm+4a7>FHUcEf&qCpcfQCK&9adO6{umDqF< z$=AJMV^taJ@zrG0C(TM=v+v<{D&R4%q`79-$F3Sg2#5z)M<0k5njO^V6aaqOg@e+l zpt>V|Ha6A;qswgN&Xq#Gy8k+Up!U5v7;ha)yFkQU1%_SjQUgSvV5-q$;feMR2TQ6_ zqdz?sn>%3MK7Ch6DQQ4`z78Sy>|)tX_LFGm0}lGwT4)|E4%0_n9R9~~T)|-#>z|(s zRv|Ncl0NXJJ)5ZVuj5d3xI30}h0i@{pIJ#OI{Bo0;fe|oHY8u1U`A(i;TnJk?!SjYDomY3iZ0qpdRoe2Qf>X+Vz2D1lQpD4w}| zL?7vv|C?90@SDW}^Oj5Q{d7ceko?uz;&ol_tXLBZ%Uw{#bpFxfZVRG>c#E^$m~uD( zg7q36r)|CXn@~U~2g4r6o~zk2uc@bR5!KR2-hPsu-0?Vuc+HXq1n+zge!H=Yy6rt+ku&DpOA_ukS({K}&6`gqf2r8(lE z?FM_g?-0lo)Zfqxbhh9SzSv4Yb9G_@A+!QE+SWn$MZ&K>JdYLlxGs4On=IfOPI7Sb zp(8aa#30UoST9{)Me+}@lG%r2hldaZ_Iuin3howoHu<1^8r=?JC8ypmd_)OHijN=} z-3lyUZ+@Y)>yWA3ijm4PhtuPVK?!P%PQN7}j2IqRX)!su9}!59FbG-51-tpJlXKGW z4{Q!*IS|F@qqxknF?R;vpORgHBnE}GIpV_{g4K{Nz+t3|z&(PsxxLSJaw@UqZVR5` zqN1i2!CG5%)NWLx<#L%`Om^drK2`QEEG~#(o3Y$vo-J~?8L@NPK_54z+a`kV;rZ;3 zsO}4APDTdLIV|>1A@SITom4yk^yUJy=&UZTJAEpul5L*U%B>Hl9Z;9xS#OolO#2di z4TtF(7Cz=vu{Om%ML1#X4nW8>$M_r?4kx=$dNae^4IpZ%!lQR3>WK2SjoR$uh*Gu5 zBe!{*&RyCOMMba!k;h}RCix&qkyi+#{sM$t)xlHZgpHsf<_Vworx46XF4;wv$A!9$ zkAIOt)$cvqHCD1y0)73dlC~n$e01P-b4N4NgQ@`i%ey~4WaF-iEh>2`X5>;C^V+Sg zm{!vqSI6k(tG0ek-$-}QLZqF4@DsAPdNdUtX^Y@*X5oFI5n~-w8OYYNbb~XEVj}!E zlkN2|7@4u#ploGwCY4h0!aUGd+=9kN6)9W8z;np^SJoHA+htO&|d^a_p zh~RB}su}lmTpm087Pak{n&-Ev5>Xn$KX6$9gcN)mqF9J)PI>|z)9=OQcPZ4H?2RrH z%tzJ9@ssPbkVDl&wX)N6veKcF^&ww^JQ~-?-60N9_i_C~3}?+@ub;i6CokW<&%%fD zX7CNETod~Ksp&F+qDjh9==mDc(es$8ZH~U6sRQ5BwOm!YK;V}c6TdolZJ*M9%n${Z zzP*2XP+VK+uwVSI7OwJ#7#3A3A9@Ue-%~Mfc*&IQ^z1r#0EE6FwO~=%f$S z745Vz{=>c~E*~BXHkD78g@5dyw76{$%R%y<_Jy&#PTj~6AwPmKru>OUJsziNMagH< z@Rc1PVzDh>p-pFq3DOdkZ??%WD)>DRkIg3@!Nty(3~;vK+9!9nx}fFOCO} zF51ELNh>9#_ufL%YY~s)nMm!d`eHwN@M^p?f5tN=g(FEOx!49*wvNX@~tPtQJzY z(`OVbE{^xPacjMkg&a+t?wPJt6et9K!YuyvS&WghX-nJT5}+h%MhkX6%q>RfjtV^o zlQ{~PI!fmG6)AEYlEt9)Z~cwYw3*ks+*Lj+t!3&{zGBsQ5DxS;uGc~`eJU#ZE8vH@ z`6*k+8i37+7|u3VYc-AW!tQh-;?w$(KMFkyB797|oaz6T0sq~70g_taGZ`59!Abh} z&ozT2Rdl{4JotQDT~N6S(Y-8-aD|kZ&=uaO)#&PZLnUDEYBD0@G0aWtQB{@Sy*Zq# zM)qw15BQRPmYF)Kg}@IVnlEcT> zDnn+#1KcN5ml&g5XcEJnvyA_PZ7EC-0HZW=Q7mn|_e6N+&*Wit@n$LB~ zI@w&VFNOPV{9JHR><=_1>64AzP0CS@@Flhf%`fQ zp5ifGHc_rK-lLW+^N93nzZuc`H}z+PAzOa0p%a$W+m?sS{}wBl_aFS@eKD+f5`|e= zZL+^#8@UP`6yAb<2VoS2FVEK7G{U<}1+PCB4iE7NJ}mw4pz(QD^KL`D{JUU6;`BUH zsM_yhVZF&iK=@n}lrKC2C?b1S#|rE6Q!uQ7Vzb%07(4;Iu8#y4z?#>ycw6xvRMd97 z7%|+{tU$LS0od|4x#*42AFO3PAM(em5=Q9ju62ih=bk_DED=~~tTqLn(GU4t^XTtr`VU*zvG_P@W;cl`SfNiX#wbpO7)v60PHj?@$3LX1^j z{*5ELjR)^fIO$v_Jt~Uv=p3c}J#vYku5sB)SU6IWbdDum6-Kk|QtgZKJ44E4rldG_}U%oYqR7(!IRCY$Up zXJM8RY4x21WF}e$riB`JlMKdq;Kr+r>Q8PqtdgLsuAIlX;J%WviOoO_-vCh-qA+eJ zkWd*(uF$E+9$!)cY-muj+BQZSHh5QF>?SttH*A^yLb?}n{&K6qZfV$>?s9=NUYM-S zHZ|3u?JSe#-B0#G3n6f4kLmu)x7QP{KnwACa#a7>PV@C(0}jcG@g=pLAS<~8U3F`{ zORy3bHIlD2-D+qwdT@EREmShv%$fT|v#`l#`pyakA;0*mAMfYdd3|s}4eOrH|0Wib z-pRGGw=9Dn)(+J{9ko70A!P`O^|Aw`{(qlJHVxLfRJLKXh8ZY2WGUp?n$8{)RIuSw z-86(`P|ih~}NA=6FIAKcA1kB+3 z{$P46cEPtoIg+xn?>fonL1|clN$F4MeR==0tMp}SkTd;{-=NsV?7@UO ztI{BLnt-zqiG$MGR@N@3OaS!ZJK>0e9Db|@+b#C5v{FmV zAF;t+NTop4-TZrsc#My}gkUO;5u7J{d>}UGp<-ee!BgVW{Tx(+ZDp_XI1IK?V;?qj z$6(`r2ykKQ4n)8ajqv0@K@YCik72p%$WruxF}mJ>iiYPY_%wbcWskr016g1jr38(G8%^&iA z?t%CHQY&`|!t)-J_NiO?Wc0;{KcnRA;0>w@@f)U~%Z#dagcjHN|B#Q{SNzSc$#K5v z=)YID@g=dYNd66FU8~v}nLul4gm+G#=w1)pJtSV_RGuLFNw&I%SgULoWyyi!vHOvJ zILpypkT~!ID=-|kzQ#{4FE1yHgVyMyKK2I|itfZI<~dd}r4MY5-r!sh47oM1&Fc4Y zqv%p!Tp){s|2_Oxm$71}GLf!8aB;^PyJ#3|4ewxWk?weM!SpU{|&=o@H>L(RU1lYyQw=?;A8VR=v^+e-Cy54BV?d2_MmQr2BwLU1^f{{q30l_X-EU*hI}= z#@{Tr=|E483(-?2rw2?i){fnPRStHkB4QKISXAbGrth9i{}$s_!m53&d5(pfG1hfX zLNEL|o*a+PDF6d;c$}Un5Qsi}UWhvP-Rw?=Y(XtHOYJ6}ph{l`cP2}5&nJ}VmMFzA z3n>@)6lIEvU+imHW%{&*Q$>9Z2zu0gPl{q*)01O$2YAJ9yX{ff&bWV#qk2!faRLVG zP)cF<=`1wOD<0GOU(V*N9_3EzA|qz6n5nhHX$L{HMZf$T06l+XntwdK-z4J_d0$UA z#sYAyRn2P0vrEHevvFcAgI;;9P7Hp_W}088XnU`Y%2xk#gD3h^$?2$jEkpc)9tV|(+ksDKzVr8b>pkF@@M&Yd29}Pa)rctt;hg9(hu9; zkv=6k1y9Jb=3P;bT+8i^8i#QsXH9S$pzc%boKM|eH>Z-v&fs{CzuLY6MGa~`l+i)c z#rlZ}A^xRATOaROeD#cXAy{Q#uDUA$rioE)ibY3kuEH~w7F+b|V?+2mjg=J)U&T5V z!9ir;6XIRTk6mxuIhaMPBRpry`Rp{>5Tgk=vF#g=v$(`)CKAQDJ=aCT-cj`*m)L(f ziqT2j+^=H{lkZ;(ik+r0=1k-|TA4WJhhtt&h0xifAkX1I%Us_l8U1F~Q=g z6>tHmO0EH^hwoIy7(tC?5GV@9aD8GRSNe@5>S?Cw5xeUvgX#`Knamnlj-uGw>O3PZ zijS{8gZBD}$z6>Vd+Vv@?JAe;Fz};=F*8x;I6QdT1}lvI>iNn>rucw4U?@YpLLSpa zU&HQw{> z185Fr_0J|SJ9{h`Vb(Pq^uHIexusqd4l;|(clM+M2X6f>rT?u%!LOLFeC?|=*Aatz zDCVM&BG687D2sFzzo{=!@3=IJ*tP#!bZ7GaBk#K(v2NeLZz3a#P+8e4p^!a8_K33g z%HDgAQiyC>C9CZ0y~=$n zHEz@j^7@R*Km1z>c)-t7o`SX>ZQemjwEw?9mgNbgOL2YCrRN|Tii8j-MZC+GUEbrqjgYc(M9c3;x8>@_@*d7;8Pfj*WwlN}0@DL2`qxydMoCMB3mKacADJd>iu67<=$SgOXYE#Gy>bo=jvx2U7sQlbL z-i_$7ixZRV>?$>rr??9;#h1;nLh6i7Wa}tr=z1^LkJqjnP@s0mQ4Y36c6^Gf7HyRE zv(Jb>e}pGRhSC}tY&1^Do9<+W-5G;|G`gM;jFdb0LxYk1j3YPu=DxLcu1gAeXi!?@ z9LrmIjK6>O^6>1HOQzr5L7u%SgHoAyC=rNuUPYR`++M3%ub<~xsXPx#o21t?m^P>^ z*`{kj|F5m{YneqHA~mWHMIS11HF)o`vx}_9%X#z7C#%akd<}|&y=H86_}YprMr!S< z*WYFT$TwV7s3Xe_x=8b*Vy!l%=!O@8BMPd#oc_^|b#M|{ujGu@(?x^Tf#V+%g3Zh} zZH}U*uKsM6_x8&UB2gZagn8W_1GKeyL&;95{=@KUt>9a?)ty)T^FR9;=E9(3_eW1$1!7w~_ zX|kutg7F(~I~=)`y%T5|^ZiGyjb&Ay0XtfJDBiEDrmb1(@=GtFewOxg&F5WKajC|) zYl}oHm#=vBwoYE&Z?im${`&Ne5?^yXLlu$=nB7Kmd7!4zk;Zl|B&eg@q+?B_=9q}; zU=o8*^i8H+yTxF2%dJ$Z-|yD%G!Te=YNc?Ix7|?WfvQRq7`zibtQU z=bJV2NRCE~*N6UXJj>vQc(y!wR(c&?u^6aUWunl1e9-nJD&}H+RHV5zMz{kZt~JJ* z4RJz3y>!)IQ3p8e>P`0JyNF;ktkXg<0lf0maMAvifWI8ewQ9mPG{uQiekvpz6&Y0UZR z&8)HaL_@yJRu*Q5MDHzLn!(Gn9d&PlHG5{U4mEWq_=kk(8r({R^SRlqWYsOOboY!dfR)9BHhvNDS5*UmVwdOUIVkkwOa zjLOT)%TncDii;{OCKCNx&W~Q+RVng{Tx!u=`<3TGS)!C1Xe1>fH#cIb#4t|W-y|;NuAXQ4^@S$<^_6db+ceRQ_S-+!m1Y)CaRG3h2}r^4YL&(FSDA+P7V$VdvSAMei6hx(7? zx#!WJcH^K0--AG?gz7Q8*WIxmI?GQ97hs49^zB5UDDW-+NMUQp6U~g~vNmD>CHKW= zQgc$6u*?8lJOBQWvHYA2#`O!B zwBdd+o*S-7J~ydi&FT`8n}uTJr|soz2?qQ*uGQ(!B0g!BZIUQ)%3(HJUVFs*WSxfYE7{-d zn#5#yN%WEm{>Tf2tjmVED6B;IJ#GoTR!eKyo2!D7#2UOqHtkd455){#zj?*H_K)V; ze})0LDPGurQsF*k#hUFsO9AK?tUwn97|pf%?~4$hFThL>8-s8LW(L1MIlqNfSCmFM z6<)5v?kutCoxyr3*p4OrIR-yZUxRu{=wPle#1J{s-+-a zsAXbe9nS(go{>E99UCKk9oB;wuSYE?p+U2LCSs7P>0L|6*VH;PFv^=lTJuML{OyVC zC&G@TF>-a0jnz`4w;<6+ixU}wQkud)aplH1oSsI5@12(?M2g9Mq**E~eVNJlhWNrp zrZumYZvS2UhgK^M#M_P$W~>~Q7a%0|bL%UYVKe5V8jtT5HK&KG1P@3?;mMjz=Z~2W z-g=btFy3YxJt|hsOoHa(`yVt6>JkcXqFLd5twUNON3}b*p$0n)%kQR;$@9}yIgBTo+y!X>~42(|WO`e*js^#k=MIYHaJCi+& zi;L4Hy6E77nqL06q)=)>KDvUtsoQAm?Cb%Xnx>|va~!l7 zm?Dkwe*EF{I=d3h=$Oq<{HAWZl7S&OZ5gsO@=q=7tV>FkSI&Q&yNB5 zujg%bhlrmDBhseJT*VktLM=!M^B&eHP)YC6CC0~JbqGHP%j-7UM3beV0|T_+0pM_( zaZV}zycTT6Kjzw-i|8I`?d=0+bb zi_Zg9HV=elaodBnVEc&Uy<;6WC0$Z*&Lim@V%cY-=IDa7P*8u+F13vKRv$aNy7%Ix zQ0Cu+H+ZDKY8IS2H4P1Oup}%WEqR0pC0IiVNacz&rnp*UpFIU`R0lHb9fdMzHD>v< z3ieYFL>U7*tOepzZ_v4v`B4ZhF}!hTL627zmu^?Mf_yNx9(~E>+oC$B(>3012kfzf zthv8C_cXC#Vc~l?fEE+N<(6!MB};Y$%A|537{XPvAas_z$WE~uT_-E0i9=rijgVsp-r^!)$G4QSiHr(I_4sC&~kBCP~Rz_8-C{s`P z9IfBJddv6Ji+>?TWer*a<@hxBn!PBnpw~5mwT6j9%gvTxj^tvCEgYX*27XQ@4`&vp zn}PFu9@I@oS)neX!dGNvWm9N>#f7w{SrRl7X%QKWDz=ypD4Ak31`hmaX?VS!i#`=! z={d0(Sw&MLA&KR>Oj#k8dL6B^t1HDm<4VZw@H`BjsJp4J&@!AL(exNruT@s``XvQF9s7MxN1H`a0guJ7wu6|8my2wZpK>w3#Plb=) zq}aC-(?1C`1{fbANl*ikBG7ykiqaLu)U4`GnJaRRc_ihmNu+lI19WS6LV|wOY#1f* zR^9J+!2jKO4;x}x97@~AQ`b4oB9P8%cz^*&W2!1C;YHnewFu5_OTvNJXhhT(p2B>K zTm0(`i?;I-$WK}U{mWyk^d=o51R3psjNeuyQ`c%9qT~jhozEl5(sUkGolYv*9oKw{ zkHOOkBet5Ozuysk3Giev>x4b_%=#I=P){2Jy+%&zw(@nDzO&A2|7-#S%=V* zWf2QuWVc?tUUCiv^C!W(0>%=EN|!OQ1rhk!i0~T|^`ujpb44Wd>NTS2EHt6M3HSBw zvHs>6wBl%T2%*79HalOVgkC)m1_}I&FO34)zPQZKb^sj?fg*q(>uq#)m|}p?S%&@y zo)V|jS0w@^fuGU}cu-8=eGKOOx(z+zE6QUd zMMf;#++1f{#V+k#T^H`zPZyhnets=HZI4A1#e~5U*BG8?(bfLRqG&x-?PE@7Il4B3 z_qF#HZokk(JzR~-L(uGua2C6FFUwc60AAW;gD(zht|errkROxxLsv<*0yu~52#?6WFy z{|)N%pSRg-2bD2NcfoyE3*~{_TS}(i@AEO90L4>m=OrllD$tmVi1a85oJ*`AG>+Bc z*4SQSf*hqK%enq@Cjbv2!DRr5$ro%-+-><`va8xx&6R&;Y_pigChMhyb2_qQMh-!K znBT}98^$S-pcU@tLt42>(m?R5)j$aiy1vP~=tu<&N|TpAmSsCgEcZUh_}DoL^PlB8 zA}0J5ny<@hg}OSg$EA}#p#7xU_FjV0y6Cg4srvTw4*zwz67oCKdN|`<&_TWrv7BqD z*kSi$0oU0ZdQg_%&6VEHRS*%Satx7wC zEjQ>>x`0$Q#)~q?UQrByuXdDS3uFYC`@A<@ksWegyL66_y{~WOjevfKl7M3|QO`kR z4(b(XLJ=#m!Ueb9=bCq2Uy6Ue`}@TJDKHtF?$6FO2N@RuLLp)tuqOfZ9#mJ;$V#nA zSA2yM``(n64s2AfvQAITuP{waBKeOJ-_OgI^%orS3Dvu8N$Zba5d9!FeNxvidHaFN zLvy+dS-hR+TdbbZqoY(vJG>r%!zV7G|AwY=!GE>(|5bI^Lsm?sK)IUq8RpSqHufjB zLCJY@loJo?^71}epca@OAFB*k;52xpJ+$0i{7jE7>77woSt-=fFO)C$wxU2=|KYey zp7H4;^EXb6It?F?YKMO?r|a%ylsg3`dcX!ACT4zPZ~i%yG!^*_B``^M$;sa@q}tTa zDN8uKn_7kWO~-L=ZstWrm50AWzrSB=)}ho2{}p)*p;CO*ZR$oIc3s*I3{+>xQ#80~ zltn17BLJ%t(~pC00`ItVu0!S;?kpt@SqfDf?g9RlE_E~IA8y)j5qJCOe3!j*ah%(>7=oW+ z(vr8?!fIR7HDmNY+J37Ota zJxkfx^mkbwd*W5dZ7h)OS^+AREL^kYMQ0Q-=6MYzh6^+KW&-tWGw-7r$?o!g6C40= zT9zwL{)5`9{2DERfet%IIt?W;DZWW7XMsN4rC?pE?xa$8^~Nn7UNZ&p%_Q+D-HA=! zrN=!Kub!sp&vv{i*!ELYyC#Dn%#I~g=-a59ZD}}`G&K3bNztv|X6(V(p?qa7?i5Nz zcnhVbMSoU)+0X!Kc9<68k@6Gkr-eGm+B9YK2$>yh63tD1t*; zWmr7SvH$Fpmiei5?iG#gypHnQsdgjxm(Ro)a^DA*QTX^d6MH{Hn(U=AXPCL2>CA6x zaY|pykv)i)>WP@%ohhfAoW8(QiRNNMx+CzIxhQ(MPAPX~*S9b8{#A^V1}EVS{)up0x~owu>IqD=Xt$n<8+x-hVvZTbwy*$nb7h65V`pq}4cC zy}vSYQsCJ0#P;KvVylBL+|~lEm+VHQ2`|1OYZjyUO!`>30eQ-I z9tZB94Eo$`kp8gw#XjfE7t>Znb_Va1oWL@v_N>jQjv-l(UGNg_0^U^C)*1iTVH$Go zpup4VJK|8?J=P@Kz&gXYiEHjn*N1(!g;81WPA}$>e{U~(>xXp5WtO@ftOm(*fz=zq z8BXt3ug4PkygsaAl~DTWJjkuA(Wy0DUZ{(7Sl;nS8h@wE4CLJS@(RE+N;=hpTs{Yp z*4SuISdVNn$4ke?Kiy;|yVW^%k-u@rW0U0Nvro7Q&*i=^HIdhPkZ$Y?Jbt@@RjuVh8nu_D-uZGycN*S8fG>qRnZN4qmUkLjYVM^ZgDH+z0|U#>qR?Gl71;7-SZ zQOB<Kkk3r3@($O@-nP@OGWl!GkD>Mx>lagCS>yhkJs6Qffg;Mk4Ic_`_#wR z_uVlmXuR~Izj+d)$lrIGX8W?K*p-X8A!%fPM{L&Nm|bl_9B^Iwq#4jRBdZR8svp=^ zv?iCt^;tpqWKX{j#+)O7Mz}0s6HYFu@ZCG3{&+_glsNj6S6r6+T7*D8Vey?uXUxI+ z{9Egjw}lw1<2gEyNIkxgH_bg5{o+{cJgUv0D%SB~xjdb3VBuv2c@?=Q3}!;^k?D^f zSl7mD4%GYHyuljM>|8jIMj~ZI86l2loR&vKP4b?IkdXHN=Xcm>npaRXsH$yP7Ix^4 z+vK_|Sn~ZvcJcMeM!nXUcuz1Z=cMFk-6z@KQebJ~_%~nfHMA`B%(QzVX(-F!8s7-E z?gjvgcer?ocODPQv!e;w`xWYZ;w{mYp(*^X7dy1|G_}gDiny!|Un}X<;pxly*Y{35K8G#MOg+3`JS8*CS^~JqNfqh3SlM^8qbjJZF=2I!}(rYIJsqPt=SjqaibT47AgDuMfyzv5wZZA zz~x;`UBG9=MSD;%Qd*$;kqce>CS6INfuwwywXVQNq`7aLmCP|@5$H)>sO7tQ4HJ9P zXXIK}=OppR4|T0q{wAZtld^$aD>2)b5GNDT8SwkqKV4nTbyD&u+yf|#Icawx#+I<_i28$1es#SWV;*<8YtMA9KMgL0tY6G${q z<~!!)%5S_rR@CE0+)dNsxIr#6T=;&L_^d3<@vWxwfy=aq z>DAH_nB}1QzhO|fd1_O?-1q&iw^3RKGs76JzZ>3+cPNI$j-SXR46#>C#j!D)1OM36 z6kv?PKp?xWpHBu-5(yz!GQ!-N9J=Q5*^>?Ke_q(&1KJTlLrC+QAcZ$Y#B$XqXc4Wy zrlV30ub+0ZK_Y!rv9pu~XfjvC%N^`4L^_LBocPXP=jtf0t{niRUJ;C(suxMm*v|zP zbDeW>H2iBAeDGF&ju|^=sz9fPlU}5ss@Dk($v6vc$pnay5VU{(e8(GUlolaVn>aRL zjmV~^rZ#%E3&Grt64gPz@KRJF(rPnm_`WeY4(0135dIZ?e+8j&Pdaq5G|frz6A=;p z03VLW+I*mQEf&se20ka+%vCHbS`8DO+yzCd>ot72tP{JzBQ{!9ja;8Rre*3+2P|%M zd54+}7C1Jh8F)1o8`Lh8GVT0*<4h#g1O}bGpHAFbi61>?BnKKQFQTHHteBC5G^}ju zzN>11OX-#a46fGSrH5l*FX=DFeAmM)Go&?Iz2M9xH*`_w2Y*(#6Rz~H@mkM8FBT5@ z=g%tlRLjdqYu^%~8{Q%OLH^1PT}o{1gW%sLx`xPv;p-u2I3&!{i$^6UCPwsA`22js zqFc)|pL}uNbq%y63c-BpE^A!t3kq8hJ;HibzthRQ&fRbjM(%sOO+`=+{g)D8H363?#g+72k~ov$68e7qZ5obep!AKR+w& zG+~|Qe66U#WT_0gjZeOzHBS(EU9F}axl4Hpi*L3khKvd?7#1JxjpRURoNv|B{W+9_ zU-96FQ$<_@XhiVWu?JOYPFqy@9Vd`l9%|zXo*=xYz2>vo;hI5BV4i?zSv$ zq82>o3u4S$3C(nnwHNtv3qb!{^$*YQ>_&e|*0yX|t%r(Dw5Q3ZgNK3!g^5$g zd>+pX4O;IBDJj;?Se9r~Hcv#z(ZJZ#8Wx4Zjn82guZX0{kui5_A)o7p6(YEi0{r|R z&?Lc_(*fPtK-E+w-h(vv||pJR-N; zmWzojN2WsEXCI}z%@aD~I@Zgfu38UARoimjob;aDWajQkdhGqrw)KCN@04LsLrX<< zZ+=OCn(p>Xd=UhuN_QO2W@@oTyhlyqM{l>01%9BCK4iN;7(uOnvSd5Rd^9_{sr58+ z?q-d%Xx&DC=SN_!bTf>FdHcFCg#?-TRi6KS4D{!OVNTQA-348yvF3&z>o7i+=v_on z0Wfg_DssWZ9Ht{6KqR*)?oh@Kf|tm#V+DO9=Lu2HoLe>u*Zeo3Li6U%_X$C5=p1

>~H$7V`$};lUpCG zUQ!250aJ5H2+I9$i~39&((}hF2}qN0_y6`-HNOkTYNC^qBiC-6mQ7K-o=+8TZm+RP zmqaAtC2Ncg8?o-X!MFyO25TLz!j8@tYNDek+_K)`_3dMOq+d%JbPs@d@y~1j&x5!7?^u_4t$UB7kC8A29nOm=o#ock51eA1?gV4zvlH&|y+H;a z#>90moBQ0Wj%##r(`B#c6gu)1Ok8tM1=4;u4})2QZ$9?K+2!RG9a-W{a4bu2jxj2pPuU_y-ZR z_w-11jM;fjVEMnqDzzBwEG}_9o!5Vk&%t&1Zt4^or;9O}hOO1aAO7c&9FNj!0?prp zpuM95%clamwA}Tk4bayuaYZ4Lx{7*;kxki;AWVFEM9YsCj`Ex)i%l(kOl@9$=UXFN zy=5hr9v}e$W)%DpfQ9!d_^h|u+0t^lUa*MrbhyNI+PlIV2q@iO0HbUOitSd-Lc#!y zmAgrYNYCa$o`m0jXK^0Ztpy^9AKj8)F@?~5tmRznt6V`wquhF&D;HoG&;m=Of19WY z{bXx!Wy!LxaaqrM|Ng!1!u-T6zujvk1T)h*MRJH7`SFC~i_AJ;@&$LZ*J#+o*x}hEJ~{n#u}&j&kSwuB1*w zVdWk>96u&YD^{vmHimaSaY}{Sv<<7)`MJ$9ZSC!gb0YX)7u4>E(V^fj(ZYAS({tuYg9aD{4v&mkBFI_ICdM- zNm!Igj=CETrPaPLqiS-pc$6W%sE_J$7qfjbf_|oKNBDZ0{yGTi6h+78u^2yz!iU=9*TIKS}vytJ@z$I+VcIx0)NU;Wuay#n>V^LL%KUWblecFg8@zt(Y* zr?Q+`<`?5*x2^H3L@C{|pzJ$2jf@rkHHq?Dl+aehLq|ssvMgz~Q+QAS-BXD;FDkrq zDEm)c6-20R&CFX`O)?aYB{KE2*gZOlB0LW`9K)291`eqrEgNr#FUIfHzNQ9`Sr$~a zA_ncxgtmYrsMY5E8BucuQJGxD`{u7z3Emujy~b9B(BCyqK<5W*Kgf@c#;_wC6E9cQ8VE`xQ@9ZZxaeFj1{ zdiDojZ!R=}a|Yo99V$!y1wpT0=ITqM{D(C^3&S^rw&znnym9F$NJ_$tarzt>R{_(>1_@gcetUX4%cTx2jxgM(KXyHrrDAB<4qc3k8dG)Ul z56pYyy#{P@>Icz;S0I61@7~oZ8t53T%4nvnJA^*vUrq085==ELv#}ze@wuH&MPMF^ zMO^!A+F@xhF|&B~L3E+LcotN)ux}8TR?k8zlUITrBJs8U5;A782J)Kolo(*){&$Me z+Xs2PnYo~hr*fPhXXTxTu~@%P+>6K;+<%AdoW)4+gTCib|qJ@sJ0RmAe%Tjp0 zs4qav`Ibr*=8nP+1{Cy-SQ3Fei>8iHExnful;2=Y;AK~rIir!Rbi7sNeNWGzYv9e+Cz zk>WGY1mBF@)}DVfhyORnc5Gw z1M_(`m-U7(3hpUI+sZNV5f1!}Rur-~6gP|Y0(z^}Y;&;7h{02IwGW=V(HCAr6iza{ z*4~pv?_cI{3T2(sk2>>{9}DbKLnxhMHZJq@GR^~?M{ug~EkM=twYy&LdWVDVcbL;w z1P*9Uo?hv)(nZ?4Tx_R(7CE^$X*hC4nJ4eyZxS(~&C)r1afD$NMhT*& z&oSx?CObEpDurY0-NmI>i?B~;#5cz&XjS088uptnIedbR0QMyGkVpf!0ztPV_H0!5@Oz6bnbvd zc^xe}I?CkhhbQ45tYEzBJlX2}+l6e1Iis81;zWllZGxJJGl`E`tX}*r^f708R_#DJ zYf3-mlfWApM5o0<1sJjlz*8wu2=cae)%_FLEb<18a=!aWfZ7t#s!EDkrB|6e;tlydAh? z^xVazL&le>*ZDnk>k48yzf4uw>P?erw>-VvZrFTu^I#Ghpw8b@^n^Ct3p1Z|KUvz$ zx-jk0-=d5m$N>m$#apDd6ljXYFoFHwEvtD8>IIfcM{~^;ya>xomaav0`GX&iA6DO1 zC4||2coKDmgF`$`t{;@B`wu3{5e2_TunbwE0&sg$_=oRbX&>x^0*mbCqox4#ACU2k z+xqO19?gcbk>z5ngXQz2MkH%4rUtc?0d-_qD+E-$+G$(K`cuUJ^^vlUd%N=8^%O4a zM8^$CZLky)*))oq3+jGoHsmFq95220wdLh-m~XEflzbk~szdULEipFmvOB|u3EkZJ zn2V-(1`NleAtoP$?{YP%O!0loAXF`SR&gD;khoal>#k>DaS#v>@1rb|AE4%ch* zv@$Rph)Wpd(OJxSn7bBmf%U!K@$6NqhtbTLh@|3n;Tb#xzTpg~UTl}DMt2Z13B*3H z6I1pP@F^>mgnXN|yUBZc-rr*h(%(qFxZZ95^EHz5ae4jRdge92DnGGWf>4GOeGgTZ zT+pwn*7~KTrERSxepeqtM=U>vs1oV!-n##spRIQ5?ybz5B7|*}iB4-Hx94}jd)5l9 z11|7h^-29nrhDHitX0a5Jmw7M*y*NBLCQAn65ro3Ggil!N|++Rm5N8(|C8g!De52}V_V<_OCvBE#l5G$I<82cb~T zX?E6qN32r_1vcz6@&kN5$7yn7tHp~rYNG2W;lb%M^V=Zwv77FUg zbVud-sx-GNhC#Fx{u2}045vZmf8yc~&)7nxe>L#z(Kv&gr_0o}HRucI=W+bw6h%}A zJf=V7XKVR%nNy)9#`(FD5qD|J_YZ~ltsu}r*1OtIMFOzCQ7jD+>1jBdTMm^N(CXMsQT*In@=q}WQGt;j8ne00cU0^FBD3R5{T(#!$%*PS@37~h`h!E8o zxB>%+haVNIM~{5TqXX30RTV!yHDmXghc_yh>a>`|v?G%;+}fd`LGXz_PI^y&_L>Xf zRZ5b;YsK#+La(4lmq8^6Nj;u`&8Kc~iNeQG$oE=t$j{)!E?*C0+6Jj6bvHZnOh*$W zi3D}OE{3qAQCr`JzmWr)8dJ$TRC2jqVe5 zhk@ss&_u-Voqs!Z$NOi|NbzCZ5bafzI=zh=0L)e$291>ppppo)hg-Ry$b?3{fQ)fs zldCndV_4XE5OZI~8GqIz9m}l6lXIByD|0cajy|?2Uan>4M7Uty<|VeW+U~j()kRkd z{hDOQ(0q?0`*EIOU-X914CAI!zYllgP>FF5zZUxLBpM6BN?U`J*wv7X@l^o#&tk2( za>UDOmlsT1)*<%~HK(2|qug4Yn;d?@m_Mf0XfCa@9Gms}(Lxm=O;O0IuH+S>uhE~u z1{l+nVEy1Cy~Qkfm6t;%)SVkP=@q)eGJzPETqfjOqOxBlb@-25yNo^^H~4%)!`(<; ziZ_#oq&PWvK(VMfzz%!sz!8qMF-S~_A0#sRw1a6Df16;8>#T-NwoUaiJy=+{sSf)4 zZ8`^&N;b=b{{DH_c$VKaLd>?IQC#|y)#miP5L9(Pfc?$wD-rend;z*&42rY#mkkWf zp1_{>pVm~Vq&XZN@_W8q1(Yj|)x!7-4f!X&c1u84DYss@%e*E?OZ(#Ry4kGWM%>itfLJD1u{z7VNte-$LRZvbW?evhZ3$K;yW067Bw9HVYZphh6~XI|S1@f>!pTJa(OITaW#Esc|NiEFP3RZrdrc<>JjM zb=8KWDYCBtH*azpOkOkb9~M+GVnLg@A%uuX7N}G}mdfQu))k(oGobe?LJ2-P&<@dGReX(z<6 z50v1nz>7pd8?`V%j9>wjgbOb#6Bq3$XmV%ZZO#m*teHy*Z3uRLBz^Q1sN&@S1lxCz z)57g+k&8Y+++UDE4^dQ|u?|RRT3I{jyEOwEYy~*PHMMYzE%6? zJJZdZPK4<>Poe3cjx*(Uv1rHuPQIob#s(-f&c8q)n*ZC5W=VmOYTVSvQnqZY$2F=J z{BDZlbg7JtYjwFQwrc*pz4wf-mX@6|oc4K;IVA#H6upEgc(#w}C8GkrE;q*xThjA1 zEcapbNcvQwiAq*?ppLG{^yLjNoNr%cZ{^3=oeq(1qGl#i3l~29c~3*W@<4pt;~+;7 zbn&ZoGimti-1MLe+5e3WU~pch8Ik9rpWhF(uCUz6yQ zZh>XgeeEZKEA+mVhvEbpR&F|H;b

J~>RZ*cgVyw!~w|*;aL2C$mMKM*|@f4@UhK_Rs8D-L-Kpi9gDFmil4p}sv1!Cd=0oSeMg{u3(FcmM`dzCl&-ZvvqQxc8l-DLvcDKdVZH=FX_rIs zlO4!R#buNW9c?I>D=V2jPA6SEKvN8A7|pG|pWAbRM{B>?4uAMNQ9>&psWj;6{6RKR zqiymM3bz)$#yx$tDO|;iroPMz4nYI{Jkt|RdcZK z%+)`%3CcB>^PQ^Oh^i94Ja%Req5mQoPDY$J1{NH7m6behYa3>FkCAZE=#fTSPM9mAQ7%CVXm8oVKO#0}|~-wjMM`25!%*#7bTsYuRcGhr)V%gG|oXLKTI0H=p@sd zz>rL5iDqB$`NLM#F+#5f8DmgU8k5Jd6XLh@wMfV01RfG@PUiiY|VwT=V^V?{NFa27* zH)^t4ME#k*K0GzIKviXZa)njf zESf>FT%*L4mtYT(C>mi_RxwG`b2l^F?RM7&vZM8ensyQ7PTZSn ziOb(xzzTD-@=K(|M^M5czktD=Pyb;s4oNID$k{A0E2h4z+ya%L@k;yjXnDZNPOMp2 zg96Q_zWe%k#fx$AZ-@oPCBeN2BZ*WObLu))R|%Vi3?-5~7BQY?fW~sLYoOwHDBMFZ zT#&a}YJ4=CVQm=z^3@WfjIlRfZaYbfJX=?|8FlED?`3(c89yU-J~0ilSnI{GL@w-_k_A#QPF3XBc4{x}-94eAORLQk9_r$7O6>9A9kjjI6)!9rIT!)Ur>VSBH4*@dmC}%}Nu|-RC(% zQZ&f(g_icfTl5hOYfRmu=!I2rFf@o`;^X<5ug+UYW53HHM3>S78KZ1gP0=rqDZM0O zk6g(X87j#91XgStm8Kyv;f}wH@{~lF0#L+cjl7KlR}fmBh6iesC-D@+&J4pqJSkb3GoyR~^RwbSio|ikNg^R!d?-~%Wf$o0^z|mJ<4FhD< zaEA?Of+$!$bz6`Z;>QwzlT0-9H8@+|wAm<{N#a2TEKYX~edtvLJx=x~P5PQTLpTSo z80GyrsdnI&m0_73zeDQUvQfM9sN1xFNPfkcB*X@?20h7_k>Rd5Z7M~m!^2EPf!K>T zkevEb!O9tUA{ZyyNz96+R#uMu5=Z4|IpA4&mp&VPyNWi(ZwfSjYA;~5$Sva8O%PU3 zThG9~U6!Pb1hD#GVlq`kB3g!BuWQmjIv0so?SUh!3j$!`3(q|>#UtrJvgr?0X>pNl zF3#HpGaXUDD^CnCq6uS29ifN*P*QYcfaY6^wHK|?lt3~2w*&bfKk5hd`M*Ve7D>dr zc1z!HIj}ii1on;`f~5cN=M3$|Mx1>a0rcPf&A&do86nqH>eCYZ?hF3~=1}6>!uib= z{p%l4;#~#M$!tKR-2Y~5|9%CO5u80DAF$*SIXkAjcIBrXIy*Z9hC@O_*B=uoU;;74 z---IYQQO5_fSo5M$Htlv6B92Fc#UEE)6=E8%5Q3$nr7{q_kCc(K{Xc#f1uXANaIJl z{WM|Kud=vq!)BDnNuKRXgp7Y+^UjHl7!XgI@n&jRZfmb5tp?$e22|YCuDn$V(*l~a zYE1jOFK99{GFC7oyuG0E@S%z0;-_FHmf-*69)P@1ZaaP5k&2Pr@Sennpv%|zUGf$d z77nKB0mul`+r&-hzfsos^#DaI@Y*q7=B!DD$><}k9%%gVeD3P#C;~yYih-;? zR!niaEeM9AKxUiIAcmNCviDXvED>SKi+sdvN8N-D%7rN}Z~&bBK_?YR4&%W_#6>_; zpPF~>IR}Di3A06+2bc^GT9!hJ1;YF(X@D-@7+t(JDlx#}P|)VDAHu1Kz32rzDGeVV zpRJeV@OvM$Q(y><0lO~+N0D!t7No(vj@cRBzQk(x{0yDdY4Q2=4t{^s{-8!U{cIpex{Qs3#8as`H|`UV-~0w0p{<%Q zdt;UUx3x~x5h*F;Uc&=6W_YZOs{x+8WV;z7Z8BEm?MY4YjegE^LIU;;B=5z=5xA2C)))vs5WS%zf z(7FC!p;u%K5x?Qmkr(?vzcCom$|&;c_%F56|6H39Tu7EDuhy^he}1DO0@=H%=kPad zygxrP1J{48)ST(p9rmLA5-G@4D=Y^XSJDq1VmO?f*T)70t@d1ig7gD2-C+bYp9Id< zkoIT-WfU3rl>^x57Xk)|KNe2B#((%5Q@?j!RZSA|MjS)B>JL|#kpvMGRy1sE3Ko`@ z#)19$OL34Nk-~}386Frwp!tSSF;*d_TU^#-#&BQ&wBpg*Ux*Cwhro|7Zbw7y5NSNQ z-tXFl@bw|&mRk(o7_axP0TL4Lrj~)ho84o@m$#bWfG|x+*w$z(by`+VVmArdoh%27 zGB}LN0|HQ&$j2c;kzZJ@x#%xajn{khZj++!g0A^mrSkOK4W1oD1*$c2baZs6!PmFH zhRc4oxiVa<&iN5|5X;s@K62tVAJTTKBGdJQY4#<5)_!HwKszM>r^=rtb>! zxvZ%pTJ(WCi}aXKevx}gFc{-u+?%}NA0hgh4=n3}Ks1h6Xq zoK%?J_sQkcdt?A?B6pkd@8&6QLbo%j!!kxY#qtqmR^1%00tGlfk;BWq4Lx@(helvF zmE40w`+WMl9~WTMTdbnMOLy#4_k#JxjWs|3m%$`xB*Seh8c0~J_2MqpFH!a&{-HPn$5{#BYsV_4eMlX3?$|qkt>Y+Yr`ve!lTa%UO@k5I4*)Pd zW?cKYyE2#?uhI=emN~hx$9hZGP+7tn8&1Eo)hs4jTHIMhO6_O`zU-lcY6U6#pD^AZ zfr|SZNVa25jCXLs^Bic!!UEz;QWn3J`~} zwb7FFDSp#W*BE@6w(&7bIZ>6%PreuWmLbycSy`Z<*?bBE^_Q()MV$Z&SOds*5eR=Z zmFj;|gg=`KgS{A%*yxjiy+QX9px1~QM}gV5EUFL2VJ@@54EIi&7U}RWWd^GLvNQex zRlR3Pcy%Gs2vYn&t}7wmCV^-wR%|L6t)0`qqQey(4@GFRYJ&ncGPvJg~zuK1>s&~0tjB*)^^J(8#Xf>2g zA1`v>CtB9EQ{Dr&5FHoL7xYvtx#S;1m)hZb!gUC2XYEZcDR;bFx~e?-(h<2(Q6nOz zb=%tQd(lrQqSR6G6#k4cnv5t3=k67oNj-49gXMI3urv;$4*1HCeb72no|pb`I7%bR znTi!C{?IE^hODb5W+gG~d62%564^|cMl*vAkRpwjpBIN7q22UY>r9H*4J15yb9GmY zu24vSO{ECb4CaDl;SqY6-1oQgNmsPxX|6s(kZ49vGMq5cKvuxJN6Me!0$cXvplNBGR^lMQ1WxMX-qN@s5n)jc}es3|ncrrmYZAn#b|Aj%&YIy--& zFHYHe8LK@}^@1uj6B$XLt_IfW+9Tl-=<+bmyHw2zfE8fgqn=mY{8_6WY&wylWdJi9 zJh*N>U0>&!2(u=lRMGjxU= z7T4yrwqp=8na^SW>oJpfoh!4`nsZWByft9F5FoU5Cz}TSwWYfE50Wnae*sc4C{d3Chlfjm>3?Z9*0W)O|uP*ryjO=(Fd@Pm|)-8PuY1{y7caBHO; z!2yAiN9J)tjwC*wzp04dEulVsef@8MOW5itWx3)FsLFeUw-++h4xvO2RMHxrOQHy# zAC-?nCgM2P1$c`2WM3K@i+z43LW4ikhTw`mLD@nsemE1XA$N@-Lk9r~(#UsQRZI*q z>}&8nlfvd`{SJMJSL&nz+tSZiU#ntnhjles53j1AS;YRq6n^W~TOyyWZLg$T26v4K zTIFCaL(_Fk?DGH`d3zDe%+3HCX|GVg8g}?wm5{4 zOpcL$o$s}=bfLKKwH>KKWAwmt=ReR=?Q=!q7o_IBX@4t@wi4!-q-!22apg8)aemC! z(I*m|QAt262Rt%XwkB+4*o?fkpNFUO_>Cep>USwhg-fCq!WQzZ?;BAi6tN`P$yB6t zw4~l0NOC?Q!r24KPD{M|#_?6JYb6bp+HSXEx7I;Y!fxnQvAUOV$$xg*P3Y)88#N^= zb#De{9%MD5H)v`Fw$nDCOKN7TO}9OWggnm;;w@EN7}8QMc;-PsWS<5?Jwh-$)IiF zm7Rb3P)(;SHCj4xH|tv2k=!i#(8Gb|?@gX8Pq=ip+wy7@`r#g0k-oBA3g4Z809WNR z;$^Oxghk1Fw@fD)MJnpDBvRwB^lxp^wBAm>M8e4!+vM*^lWBcB@Z9?Gqst1fGj7RV zo}#gLvG=9%0R@yM+s_%+rL|tD?0hjjN`lY)Fge9g1xjk<+D4^YYJ+qNY(fEPq@|@rT0#V*JETLpTe>!-NJ~pfC?z2v zU4nE;*O}{A&wD<+&mVBEb6tEg5cisEjxp|W2f?@9(4sc*wxX$K5qNG&=`3L|i<3rT zU3Q|3lZEu zcQ&}zYMp;)2Y{L*V^w@zHcvL+#$U*QQzGlXpkXIzVZ|OtBy3N=h5G5ZjNn&$&Bu59 zBF75?$27W8dT+MOT;Q=eb)-n(b&DXPvuMevGYe-*;Q}%#pZ7iqk4@z3D7q3;Ty%5MfK$>N(QhY%jozZ29<6e zIpOPR_ebeVvsd7HJ-vFd%tg*!HwQm9T^tY4rT(%aN!&DZYE7G?C&HTKcwpHNW#C zi{@rset)MQO+h{+0-tpVs>j`Bjy1&;i;RVos{SKo@Vun_f1}j%WA*SZU~yD9AGlK7 zoUzu@YNUUx?8J^|zNvXst-)hP7&nr~;XSQ`DlLJ^_O#vaPE$ON<;iyaWNbI@_;?X3 z0a$EvG86|S6vh<~f)|+7vJLzSWyE+9iC>v|oI7v^vlP&NOjx3i43~jZ{8r2h6UnCo znEnz!KDMnaDeT>nTv^VE(r_Jed+qy&m;{nsmZOM$op28wKq~G!*I@DP#+9+H3A?o! z)9IH|wap+#8ue&W_eFT=nk7HiVBGIz=A(`%WK)`Jq==9p!8{ug0(xG4#J7)c&-UVP zYTi$YU7y_4R8O7gV^r}~>D(x2X4@~PkI#yR0VH$LJlcZE%Zj`j>5gX;6%14bf(KOG zeT^$`LvCu=99M437IlA9{5rN_msQ%bPP_B18_I+e`2xJyhxSuL&rBJ&bdF~HK;IDX zl26&TN+ia}Crn}%^LGi6yV8wL!Gt!w#l5yz zP^>3vfw8oPK(H7{EVFGZU6aXW#yX3*{8%^5CZaNWPxR(&WetoEcYCPsvcq)N$!mb| z80Dx<5i5;ys0h!SRF^^+_19X8>M+`k+^Uq|?CA-L0I(l`Hg%kW z91(>%AZoXfI$wOgT5QI<0L7JBjV3-@3aUVhpf%(JgeN%SDnk@@sd z02ypD%~rcp9LK=KA6UH9)^1L1?K0%y9DUa}2Mu3>T_zVDYZ31PbdPW4dYm@^kDnJ2 zzwc-8$Q+ulBV#sT6js-r6NH z@Z#}(Dn&ycuR2=534bLBjzcu94~=_^G^YWe_nB{wmj(oMRx~c!t;Bu;=S0jyrO`G* z*}9B`F=@e*ochqTG#k9}(sbI$(oAP)U#em8YsPA;`|E3Dg~sJkcN=}5Zg zfLtEWC=51#5uSqsIpX5g$`e&RJvaPOeBqemJV+X3b6|KyQw_-c0hXoQz>G=f{r5AG zp7fmR&b^vNB(IZ6^D}JW@p`7S2K9UKhgXh?MZ|uTZNt1iCDEAUL;+eK z^~fXM&xq$!T+v2(WX1=+?4@?+)i!XH3BnY;^~UUQ@1_;lZ9-TY=_0C9q!s(EliPvH zXrM&BptI{Q2Q7xWr9dj?<(^=1x6$pMg&x{2%L_G1!q-a3E4{G-(3rURR%)8@HhpbM z-r5Z?e$iwx#FxX?XMuV~b02o)sQ#^HaDmwl_nrvmM_7O?!=lmct_hU4w+0lKR{J;d zlvdmZfO91IrK}mx5<~oK5yf3yaA&WdR=}hSnfQjE!Wzvuxsk2;QUvKA64TCpBT#Rqx zrMk6AG*P=Cl#E6BK`@U?c;o9~N`vE4Y!>h3pZbGdhjg)VSyYO|_DB63iJqqjx)q_d zNn|e&hQ%P0eSP=ZajA{?VrOOTx{6dFWTkHG5#ehae!ne6CbU*06 zcdBQpg}Rw~XKCM~2;9ULOA3qK{J>(Nuiv`ikx3`*y_*JwDURwX=o!fIe~-c(XGL-{ zp#^Co=A#G&yre=PFbsS9^|JVP?5DC}zKf&Y^o8O!)xk)K2ZRlPsKZE;2>=R71z#7QUVx@C7P-MKqpu|kae(d{>R37<8@OhF-QjY z0t~8?=xC1WtsR!(WT^5Eq30-|m0VlI-9|;YByg?RC%Rl=Wmr3+gYNs4e_;`H>eZBy zJ*GJTKO_EyhJkS8mrw4WF{}R+)-kAAqqd$c<2Zwf5dD00r^?X>;-!>2_0QOz{pppN zopPM%RZg6lCuvi4&jM2bD?vMsGvG@U0ew;Y7|>lSy_(OWH%nx{X>gomQoaP26I1-? zp&la_*tq4wKZ=$EkZ#IlbNuVj+C}(styzCcyx4UWu`W5!6C}$v!IB<*T=)YkCsZvX zAJ?GNI`*iA`v<*xG<)#~?}RW3Q34&Cs_ub=rw_@6G)Ldre)^#mZx(D`2cV9=O~}Y0 zU?cyG`l`xmOm~3O;3I&&TYoSAjvW0-%H{Ayy;|#BsleawvItgL z!*WEAPsiW|@>YFuTf?&D8=9I6q5wr=iJyTT9S^udxbcEevMmcSO}j|^wi~vVivX7C zC#xYc62hwrO2A#z30({7Cx#aQHf0f+11%1#T1g1b4bRdo+>Cq0wji4SFoSvMQk;}; zf=Jy`CYmreJ17|nkle1Dv2)`eW09FTuL-PP1JN?oHA^hnqYyMGm~;Gd4bB!e#Q&>NRFo8 z`FQD<=rC5x3Z>2>sY53sB@9skVaE>@?W=()m1Uo7G=K8`DBFk-9ZhGG?QVdZDTZ0;{+YsKgP*hCf}%ovO*5O&{$Ck&o2i11Cg zJm+IUM%;4_PBK$iZj(E_t3dI+R%(?fd#=0;;0U{A;N2<0Dg(sa_!Z}97=E>Yz>AOt2%fbaYXps4BA1^PwvO>ni!>8Bj5v74XeHS{q=j*Dqh z=V7i&x1Y)~W*H@%4cBpqllp;i41MXuvyLf#;&4r+oz_}6+q!bh>y>j`VEXBKqTxWY zRQ5P~MT=9A!I5rHhWNyUp7lrcy~5q9*U-f7Pg`~IL)(at3qc}mPSxtI5OJ~rRt^Zu zlvDXA`kSefoaFc+zGtKXt41hht{YT_ukB?K^A_*=qA6M+RX z<)Kyc=tq44cc>*7K$U;u*he3VI@u=r8vu^8v%s>28DEvF#k|RF#|knCocWw4?E2mW zwH$VW8(F6xr(x#v;Daxdeoe zBvyo2p8f;UfcFGYGoh$hD$T~Etxyf_2ib^OqJw)Z6L;)H>)r@>gUvy7MP>L*N^Jl_ z!PyM7+-??p50I5p?vy4mt7cOPwMn(L$rpU_T1CMZA+YQ6dIf;%u5&43C{O;?ryf1j zQ1eayJMniB3pJLgtP=xAJp!`#vm(UNxdhMdnz-o$+(fwM5KskBFO?H->_PrjPxO1b z7tcyJ-t62QrzwF6<`uu*U*{lN#h)Pvto5_rfG5{wN*_G{$!dpKl$A0CxxIgng;W}e z7s`y*={-dLB<)Xd45ii$WD4hfzt zS(HCx>Hud?E>F2u1vMQOaIn%#DL%>PN|W#^!?q5~SAH614#WhGTp=FwBt(a%vE)3= zqpyZVPooB`Wl^otBueL#S}q%9F*KfTS9aZ%QBBUA{_J(i4}1vRr&3x+$*6Zhn9Gw_ z@yWpYj`edwaNaPpsxiQT@J4M&c~JLv*D0vU+#zwW!Xs?GFoX$|fVpk6UuvQM_eVIg z-n?BMnZ|3W;ZYkae&b=#t0w zK9;zn`V{x>E??}uejsquX!}wmIt^&-R4?z($rq!j^uXu?BlDvo+}6 zh*x*iT_E2twFCn0hQ}v>Zt3>5MhZi$@P&O8X8hF%foq;U{$iK2wd#6YL$^r0J>S{oT-$d6Q%cr5<0y>_h zgTsVyhPRIK^>INrphb*vi9FZ^q|zvk8lI?bFl`SkcCUN`;KbIDl^*J5;2rTM$em4s zd?O@_g<53kC<+gE1)*IXsFC2Puf)aNq%IAaRgjB#R4TAZL?p8&PE4|6s$WA^SgpP+TDz7Wo>M66|n8f=6AC;@<6oivQSFbrR z7?r_KTgO+_$bMj#gii@ESY)E@Jf2?e`J48}@D=#-3^Wgc>{*?jDnQ2ZaZ}&lI3?6A zB_}K~L8caV;R2p<&ph5nm4c09i{yjy$_{&Czss400R@(<)rXIyAHkmXyMQ&CTz*(s zx+oxR6X8WywB0lb0hS))Wru@gx{AUt%&Vn(%DKJrH_0=xo__&}%yRx$O$e?CV%Saj zs5yWWF-0D)OwR+ku8i|21hSX|m3B1T6t2V1Y6naiFMxgt;_W~Zea2?v%q7RJeqKH;fMJ;cudI6q zP39^WgtCap(Rt)wHqxsVTuH~an3*k z-dV1#Z~JNSXHPTLUV`+{p-QMbZ(E*?;h96wtl2s-^op9W z%yHd3JZv>9%5&FeT&wuh!n}3%4`2bFbp9y!Eo0;<&4btROyk!{J#l>Pxy87@ai19E z#h|27kCLIt2dzxS+ zxx-P!!eB(%&(r_L`W(laPbi-L~<7=>M;NwD3K+(O-hoH${6x4!_8i|C#njF_}aj;AoEzL)PMS1;LE#^2f(!y`G zZ`>vhf?w0)euRhJUAF+pwn3UtEhCmlx1Mgv!QjL z&F_u=mbGacKWheb?;V@>s<_147NFyh|6v%d$O5j|!kV#^zlM(I$U0k~F?4(K9>H5Z zs>q(EZ`UY;9?Dw@COj%+@a~t9A7^M=Oeeh}BbR?JX^TNlMHQD`#*X~!YMzn(cL~X-wv6OdSN2jz$MVL5e3Q5u4sKf$ z5000JmlWSVAq_&wN^IQ#@UIGWl|KrFO3tsA(;_ieKHdnI?=oPNqC;jPqRSjs?a)xv z_RzFY&n7^>0#b!{ze|3&bZ}|FypLaAl^^(D)0yRD zgiK^psxV5AW2yoYjltWSjjlMiG@F^^TCD`p}|KJ#c&~J{ho^ zEtr^>$#p>%F>ToO13q;mK9^|R|QWVyZs!)Z(M{8DVk%%v~1z>^SQDF4e>fVrv!A9eaQ z6Wtx?rTAaI;_#6GM7dOA2z?m;%UAACVz9pWX@rdYzkIX5e(RhdZiJpTW>e;W{eA!r z5e5J9|NkTYUp~(NPko93!Qf#d{htRG6Y~nReQO|w7taz}Vw(d7-;%efGSrE#fldwA zy$xK|??7)Xk~#Sn^A!`I?c%=!Fof_dWEUv_bRGEX8HZoM_TSDBIFIbZRE2)QpdFt> zYon?+a0yxieB?)vrNzBmK`#*k|Ot#=yVq(O~r ztX3$~UoXq|Ow7LD-!ofnvp)Q_-~y{;M&ugv!AH<262xQov$_zPIfH&#-fIEEGjq^* zM3 z{u=1*_&{&ldJa5O$l;!#hvcsLLiF?<6xfMw4s`uhx~@N-KZ!{%5k-zNJ=I$P^r(#J zk$|l;&;et$B>{7wufGXKxzYn ztij$J6oNHq0fh6gAy?Nt%}fb+JD?Om?zzf<9x^zoU#H>gnS~p3kD@jZgt2zH*M`5_9LZ4~25l4v zKqRv1gucCK%}?`;wA+h{Teb}ywJBAw@BdLZy9c;`Ba6NOvWnE4(X1xy1wUxp+sSP} zlN)SsTp6If*$-9R)$)l18%Ai!s1Be-m<$5_zuOFKvY=@apxE{fzu{PbF4vflL=0z+ z0hU8YK=H%>yB^rhtwpexzpgaE+01B%Dh1pD3T`<9j~^yN4)X(wLZk2KU6>KukntLq zJPg2TGPxmGxg7O|<2Vm$JFoPoA31i5(ZV=Pr5)yIQx^N3$h4+37bH|%No$u zFws~PYVR9%)~p-dUT7QpcjkUOl4JPis*hFxw?9Q<>&b(E?|wB`WVp)yG>6PSK@5*z z(r9X3sVy=>D-8&ve_9r24S-P-o9*t(5bE$6B&*s6BF{CTVZwd(>C2}dV3spOhC3-( z_ACI2u+_O*zgQi3J2|!sfTTUGU|B6?0pKL)jDNuL_T71Fz@Znw69OE)0BC@RoBii! zCTh?iL@`tWt$iRmMh+GsEtn#*Hs+e^-vHL))sM#EqbNFM@=GYaW)Mt?b`)F945Lnv z&B<{5A7kk;`F~0iXbWNPS+55Q##;Zzb^a>h$f&!3tSF%n_sP00k=F-V{Axc)2%u<# zHN#?*UezAs2|O7hdx#^6#wTo?qyBOD^CbCy9nS?fZwiJd{JVUh?<@*5Lnco#pHDA4~X-QHY3jnOm zcvyS1aMthkIw15AQWH+O>A{hs6&AsknIAGF^uPox`Mbp#N2oMgS|!6$o&~j?IxHXY zdr-7f44l5w9p5$lZT<06N;BIdWj!i}4fz$>&(TgIYd_aP@632zoF-Yi@a_=wvgrWb zZVCv!f#zZ|XZ~4-uBooB%Kr0&i15q$Wc31Bv_x>BH@!&OI0Bc1ZJZ+RN)Lco$V6!2 zvcv9c*o34Y(%t!GI$OQ%{FrrFm2BK*_@DSb$Yuzk(+Nmoe>Xq@J#=99F;$xs`|n7+ zS1|^=Q7(QBq;4cbe~{MndUM4C06T%+U)@kfEgBlzcAR`B1dw2wtsMajg|O5Iq@@xu zd(ZvY>6CqKaqmGyfLwp4&kkD|F)^*I1i21X_Z9t{uptI#L);%R%j8dWH2(Gu61rkG zQ(TF%X{ze0P^mh*1cM)*W%jJp`cgpQyWG2|r)8c*8f@y%I+U?cZvlsyrNmzSw}F<9 z37Yda*5Y?S>qZulPwZ-r)`jxTBdS}s5QJ^hiQx6csxSrL&t4$5vlF&uTimEx52eX=%{kH+j-Nnp(J@dLC zx^t!HPJo4{UI)q#>{7s`v7mnZVVGIUACqj8v!QyUC<7hsYa}_;fXD0Z2*rwe{YzhSlIJ(#(1zF>jURj>&dPF9?*n-~q< z#T##VC2p@5ZeN`61p0ElzF+8!gWoX362=8kWvf*V`P*PA@egs24gf!$7l&iNIX&n= z{*PoK0$_{g(C)%zkFtW1{WH0SfRo>PYyWk&dw|AXPUa~ngk-x*peFG^l5W8YtbfLX z5@w`A?f1;KJ5T>R4C+{MFE?6_hIjzt^~`}v`f)DoA+uc)3M;Mrip2bm6KYo{pvVRC zszMD9oW_7&8Q<^(VF^#dnVn;{OJ~SfeaHL@IauYRT4Fx%UJo%?fuJa91=E>V*V5+^ zy_}ZCMXEV8aJ5;nKE8S2$6=fUmz8JLU=K{d431@AZ5?Kmk&PsMFl&dSVu~OKk}$O| z6ii7UG+8_Ytpmml$y%$aC`+^953PQIw+X-1xwS@))`$0M$YzdsdWl=f&`U$6`;CpW z&A1SK{W+*s1dSn<3%B)B0Q381}(c+^_oLDiJ0Nm=3eg7&v1&g&#C+$v?E{J7nq6ztE&#^7I7^ zUm+4Z;uRxZxSKPqBKJ z#Q#l^0YsR&ElU{Z;1E&T48|USX75uj%j5Zl@)48%2lX8Kj*^>buxG9LqoPdt>}O9# z6jjVcEPU)0>pzP}7vFcql#QX*RT`E52%rd;cP~em4jBSS~i@=UP%Q zaj*Yk+l9PPBo2C9fBA{Ki5v7Hx|i`$up)MAW$(_&rKlzatxmTs2I5d4f`I)SrvGT7 zIIaz}$M5H;iJH0p=<`7jB}NOEuvH}OUZr6R4mhx0C>aIZ|RohI8rX-Y?)~3+$2Akt3YubzHn~u}(byl)DgFMnF zdL3$O*r!ooB4~)@ec^wHQ0hh$)pP|EPDA4l%O7HdH%k^<&63Z_lFzcak+r0}3(Blp zTu*xsL09vbYrLud+Oj9c2cx~R|8vr7s;IQ?w_Ert*~+ZlI>lP*>Fd%}t)IZH{ntoe zO>ic<%hj)C^%p$&cT3@_*sfglHk55XBp4Nq8e5ay<6SBnTtdD(pttaOe&Gg=y;X}> z&bK}Y4ZE8} z+D~7f?9M$-{f$3kl&6~XL>d+LNbqcC#WJ?{9PGr@HQsaJ-i4oXA(IeC$&j&ZANMxU zSGQ}E_8fQBpwdTD9v3Frw0dO#JHV*OXOugb_Uj#A_+3Ptk zCYlA<+2B+8&(L5LI;UO{+WEy2^~n|(I~bM2DH0JZi!R1H&zokoJkdTf9r}0AFlhK0#V3a4}y+VMY=|F8syTAJ2xgGcw`Dv63dcH2u z2V1~J6mQ|o^PXD|o_xd74j(>PW*RtvMS8TG&I=TkRexhM2R2IRoQ5s(>gWZy-8j_t z&yt9wjb4$oaH@Z4!X|{3z+?6`S(IUXgld{AjtZ+iK#dYOSBad^)3A9q#v9BCcdyt2@m5aU&~g*b*z511lzCsg4HP+c!*>t3 zv>XE$(+0@uOLM|P7RkSQ-7yX29unn^bl`NyOCkG_;b|Mk;XSVP*E^Ui3^2!%yB0hD zp<=nRnEOv?nU(l7qmG;$0Z|xKmYhB-kC8Nm$mFrlgL5{KjOA2P=hRX-3l|L z1dwc`-O!%Fws%j1r6_&jXX=WH%e~kkm<&9l<1AG(adzP@l;ds^2i*;oQq*WLH)%YA z5#tJRhL>D+`(NvTP{c$nr%QRJqmE4RccMJ@7E;to}{G#lTG&uT@}2)Ktl0)X`6O z9Ctn($LCYt=974wG-6r*V7#Jv7c`Ep7ATCCuOtJ=FF#K+lew{{P518>v-(`%9$ooz zd(46LHT}a~Q?gsF>p_C-VuA-N_d4wWO!Dd;*@H&a=0KAq(?7xM_@`dG>$5>%KK^V` z)lGT?m@3W>nmRxKH;?~d-oLDfwA$1CKg+Ewga1eew&DojJPr@ZiXFgev%i(W<-)^9 zVVqCd;YPrE937*XMuvF=xiugFCc-}$ZM)@LO{)zTR(j8*9k5d__vY#6FbxL0zN6UA zZtTXb3c$ia2Y)A96&D8(i|y{eMCJ5b2%4QM8Xaq0#Sqd+K*()Q7M|EOOe;PaiwDs} zzY}Ne;E-`B>PRIW+#scH=HAk{RjsmOrsMTqD%siEjSBeaHumJR(NCLo{f}%J z-M0B#t0x}DUVecuW1mLr5tMV{Ch6YDcRaI|B;1q3T-DtdEU$oU%}&R@IM_rx)j3DC zmmAZ$klZ4$B6$QGW~MBo@W^Pnq!A?fQ3j5uI}ayo|GA7c6Tm>7jQH?BID%Ee9hw@r z=Et~vbhI!|j(RB*lhK?c21HTb!ytC}g?{bprH2t&Jx4$JFmr8OaP*rg)n`*Rakcgp zWrYmnxfS(cvmPxLgh^R;@j4n>aZ)VGwa5`quos$^J>#4EKX6B5w6d3XPQJ)K<;B*i z=a&Gs@+dM+RoCql-L)V5yC3_4m2fvd-ekS~f*Aaa@}c~@DZAz!iZT-p0!MYFD_d5B z?B^x=l1WP3^fJjADZwtU%2U0oc5ptjSOva@vrB8k6u+9!9DL=@;{qh0z?_SL@|qmr z`B2-DyH1fEX6RH<4%ksmedl>oyYQ}V;yjwT`%2= z`?-bBPVsu|A7Bn+mRE838KnLo&88Q>y>fn4UmozVI{$5215A$p<4deLzA4mDG=@~N z59O6c`IQEa(MKFC3}p$OpMKAa%RjbYoM3A-I{p`jdI+WwCriNN`4`XmNgxp*7M;$XiI0N_WEqmUZG zbRFZPLeNVFxC_QpX`Ruqs^z$q&Fs8C4j%~!qw%7Sdl_E)EVx{}+^pW8chX>fz4u`1 zGS+N~qI|m=p?aFFl2L0qc_OEoPFeagwbw_Z7}Zqs8Bq@Hy6ox;!%iQZf>bZR*cpq_ zn-V0GE&QMS20Vzz%dJnN8E~&#A$BgF2K{;I2mr@j{b~%>xdj~_%)rHlC<|(7k>a;h zex&{LB6xtpwD14=?=l4~#NH4{gv*pW)hj^x#AWc+DoJuFOzJ1wwVG9;RRMRz3q7+E z(HyC7YBS8Ss<%x5CXy?%Ym6}{K*_$gjJ|6LdR*hMD~AHOJYItA{uxtu-0{PqO=V?U z{xqm)y**?A{UO^}1@F-*Scna|O z14-K`;|{+YYUcs&_`V13FCHk(Q1FrI6Z8n7y~UZ!;ZkZ?mVMgL-=DVD>^ipW@R5b& zJZ47Kz{S9z8DWMvW=~=po{LEr0YBJmTlECuAbEMWm7R~em$ToQd(iRxx_gJGly)A! z8aqB5QFau3NjHSf|B37_vNXH;LU*+Mem%Rzyq`Fo-c#97r!Xu1zbTSs5H(a$Y3DzP zDjgN6MWa;G_!&HYejS)G;zfK)$PVnR(y9X;pVE2Kj@!xD^Qmf%4^C+YORE&_M@D1C zIW^PK1IFUPNuFh=-P+R^TXS!$IV$_r9CU&sUTAHP#lqX!Yq+ECw!VBEDPU9OtfZ%^ z$Al}VrFPM1S-X8Gl|x1i=dkN;8`?f>80Ki)^4VJf0-PRK^5%i(%NM1ZF9jS(&7bdf z;m4d@X7E{S#kF>e`&!(SVY#N-Co>-PEN;$vme@%_{hRog1@{W+RwzEL-Nng{bstu& z8hoAvdoMxn{REm>8K1C28LUU8N!-!C18x#*%{PqPRcGXtHTC9J%s7?B=VIsjCZX=b zAE(`J1=k3+&wCu#mseB#TcY{jZ@8G7Wtpb66gyN8JR%zQ0Jo%SCEdB((PX?teTn|r z2S63!mK-ORHO|kdmhFF=)va_Eaz96AfLZTv#E1^LkqGP`wKF$?+~DthW^Oj78s-EA zwc4E(btP0(OI<*|?cdh~l;e7brlfE{nY&+2ZW&ySz2O3mBu#beNXO}YBd~Sh*iIF1cV8)5`K`qysXYacr6WUV^1IN zvK({3^^^k~#ThV$EjI-lqULO^Eh)^&K5Y90{ZSxeU5QxfA&Glvp3C~nU@pnTX3uE7 z`(m|Kl0uAqm#6C?`=77xVlQ{kvL6&n%}ibjM=^4_S6FM@`^Ds_=vH6yrty;AGyJzi zx$Qh<1UW9${*lewtlqn{z*VaV7S~Y|h`_9{n`=sYP&dW!LzH1dgAhwP4WNMFRpTThF`ycuqvq%e!Q)Fr zRvXA#y7HOwxy*rwgrR@cr+CV`%wE599Cv!&c%6LtVkGs}XUP(;OHih$*}ljpIH|4m zySLB`>E@(S>hXn{R}NO@3T)=4HDPV;brOQVki+5%!7cn<;FQ&>{fu>RNN^>J&h{DW zWyJ}&0zorXaSq%9sJ1Bza7JE@yEz0~Pbu4mhI%y08rA!`Pp|K--MG&%DLXGXFg~iD z7ci)Bcc(R|0XjFgdwgFlTlKTs$^*J5==xUPUcVN(7)UUD{UwYL4>yObt)X{cuIU3u z4;L;C$jRUt_uhX-?_@(}1YX#Wi?PT*X6qqW&&53uDBMrzztdk274}D3ZF8wn*mZM!A63DYC>5Q-=DR$x*!B|(3 zf&S!|+4;Xta%w+Ugh`Dku~CWWvNI)Ll(YS?FBH^lT5x)EHYq$=lEeNv$+^$TU^dn} z2<4Ir!q!BZHCRm*g({?Gj71v2!SCv99k&$(fEYu>48YZDC6P9qqGW?(dR6yAIvACw z*=LIV7U(+BO*KAy)1HV>91(X#Tf5{=NE%9s!i=gBAq96)dQ*#yMaYDl5O2UXF=!al zCnuN>&)rie824!a3(>(!ldeEs>)nf&W4=wCx1OAJpC|rZ<28bt5T2S_ckf@|sl*x? z-dQ$+q}m0cJDL%WFW-3ly!iH~p%%ZJ?xL}L1%}B{FjYo!vf|}Cy*^; z*7A;*dIVPY2#>oSV+8*y2AJs8XYc?3+sRaf?9nTE0$oX;(%R+X{r(yMgcCtP_lGX5 z+;bwG-1F?FA%lPKFA3DBj*OAPkr?E5?DOO!xQ!AvmiCp%0i^m7-;oB3Z3^BOfPKl> zU%ks>>yV)YDj zPCRSZ{YP+_KC`!W48KSSW2S_OQoAp8Bzb!z2vKQ2eC3 zy298vhE%}bH$V(r9h&&UE4vJ)Ump297odW{=<%{j-)_6T!DBGDYdPo{?;SWdm8g-j zVu|?q^xvC29h##5UuZTn)_obCI8RpmgJ;68l<5N)erR)Dep8Ea28!@ls@+J=RW2dJ~^T9pXeI$jk43xGSe@53g*%?1*UQki}qV@s>IH5>7k_;)oc1teDf< z0m=ObT`=J{SMiU!PWT7bJs{5ZDu4XOoutid@Q^rSh7RE_E}N4u)mFSD3Ta8M&*Yv zq_hG*cF}71hY|Yn1~bGH3-d%}e#f;a$)8%o)xhA5LY!n6c(VUB(P&`H-5XyW3Ut3pWY`p*Al{<+W)iBV-<#=9dP#(@TWe<8t zKf_0|{}q6OLE(NyX!dp~K=VN|LA|VGWB0uJ`=*h*`-og#b+I38bUTmOdVfoZx0;~h zJ0I>p9VJ}!6a;=J4AI;tP&RVBnXWn4ODIV-UKz-UNy&TTllg?ZuDv54)P+&KD3!&? zXTASViBhu#i~-GzhLgWVcz|f5=8USwpV;#@4CB%r#4riSXQ^)noc`VmY5HW^c0Y#)NtSvs7)gsJT5Zo<;fLK0j90?@${MFmQw^lLhC6=m|_#=jmrm$^9 zj5$#H$Q^;^uTc19(gftofUu>J#Ss2N_X~SNm8}qDpuxDseT9d3H1JxDss7IkTBQ-7`n!y`DZHd_%--u&}MN}z(Vc|HZ;t@yVwK-4fBLtGG2fq*suc)@a! zSlB880>^-$HUTvgc6ZZ0<(B?MJ zOd5V09qE8~E_VgyT);L;VKAzNL7dFq?$rojW12o}SPWEJ>826riI+mFK34%O*B&XW zp|exQ9$}%y7fZs*)&nl}?EE`yZxwV`iapuuKDT8HQ;YH*ZJWdk zgaeP_i%I97qLLoOSkl)h+F^QK8uR5=Y0_3SC z4gYxzuL-x3aQuyf-UA5jA_fC|Prhi)^PzbMGofc|L^&)4-0*jJIQ;??AzTYqI@`R1TYb1Cp63#gSZN-9y1z*6A9`fy@d*s#@G1X$xD ziE_OEgc+eeHg|unD=myOTJUFYL0p92Es6*O%^pBbgTP?bPGEy!)$fru?A<-37Et|?qMwM4c7RT4LomtfNPfz>iAfN@ zEL-WgbJ*(>)PyV@?Xa*Y+(9tlI&hu(n>;GnQC4w!%}nB{NI(>O!@Z^juav=x#x;r> zu=C)bdIebL9X>6no$A7|S8OmxE>w=@##A~Xd?W-2j63^Ey~fMfvMnumvUazQV`L~?*!nB%r^>q)x@fUZ zyOd;j*D&!}fvaf|%ZpH!kj+|4*U>*|a+KDu9u`(0O7MHgEK9iz2nT-^&6;`}lQ~5{ z2|j^LrqJVw?fxj_o%}EerDxg_EdU(AkT%}s`ACmeNFrv*|BUkAk3UPIOK@)1J@4*+ zunZ_`97y@d`(ZL)leN_JRMUA`_rsKG@FLJfiq!Rr4#=33u_83k&n0$KH!KKALubYR-Nyt`+`AM4V zE^%IUiAxUL6EjU>BGH%gHs1)RR4`_^wUh96~DkXd2RIe);zD2Ue= z5#6~7_os6T_wq@HV=!QG3+v?-sg(}9&V>F;u*%g$j&Ok|XV(2k9w@OyM(8*u-+fcO zcQ6*lm7}Fr39&r)-J~0(^k_!&b?B*4=erR`xw%%XiYzdghe~2hh~`s7F5Pqowk&?q zHJnLaKj7s7kn+cPuog&w{l{}Pm*vuLLQ5N$fI27#EB({$E;Wv>sDEPrECvR8TDKjB z-cR6?A+^fhY63Li`TgET2SaK5+@z@f{^8TN^*rluL!C(XUcpAk&DtMlEiVDZ8jePJ zL#dvc*2>q|m!#cP>US)I_tLh$Dv~}y`zGG=j5TnwDWgK%nl=iJA*du4>E5VQw`L4OhV-l{NJ3p>@cosD@`l`9ZRfbVE@5~t^*IK@pZ z@VW~?o%CY5&&pM@UK;FtAr3xyWp$N<^Nu;LMsW05HZWT~e(O(ol_e z`~JE0Cq66IB=F!|jJ?!MS+DfUB!j&Osk5(F78>&|Mtm=5#W>~4J~yC5{5mR-JriqO zIP7(6H7Wn@=n#9~>|Up9SYpt>T_HteJFTzR#<^%neN~LqCCUHvE9NS9emzorn&jni zQFLH65E11x|_R-56xT{)-iA!Jp{*Y(P53AVfUjT+#XS_q?bJYRl`iTl_fOviGoq`8HC>jJ%Dga_*3~wk- zTl)orj7(o}-c%(&Qr$tzX3V@d?7d4V!tzvzrMxs-)GH*!7x(|!S&@a)VO4&zpvQZ= z%MtD?{(>g{q1`WI1&3FyL6`JX9Gjcey4iE5?VEfn?D3vS-(pWb)oA~*ddY%{psb#@ z7U{=Uc@-iTRcpd4V!ba3&j2V&8I+34S4_s(4wB&C#(&W1EMjBEex95?;JYi!o4$)zqgU`Xg+Wu>fzYnbXNWn;2tk4My0hYPN%e5SpZjBw#~h| z9O^9-F5oT=Dc0R(TCk@C7r0!?X)$)_t_fg*9$rsb+;*BWU$tfOR79 zIGl%hD^D9{Tv$-B&rtou@_OJ1TYEcbqtPa=h3&GLPApn34N-i+#zf|g-@wz=WHm+P z*Ij6Vx;0U2N3S|R));UD<(T!{0gkkY*tKvFz+{~`ka8^t=_=l_w3KW zZ3&V=*AyzB+%7*KepzXPd81+Sv6fi9?RSbEZc&2`Mi>IS+ZSeh*_w~;fs8{ecK?1P ze+G7ekI#DuJVQ1U*nbe;z0Od4Th`T8hUM#PyB)HaK>eV|#8%Ou?cvT)6IO@;1ACQg z)Njn~JseY(Tqy+IA8-bE8>NO4JX(Pa-yUG@