diff --git a/megatron/core/optimizer/__init__.py b/megatron/core/optimizer/__init__.py
index 5ecc707c..dc4a3f12 100644
--- a/megatron/core/optimizer/__init__.py
+++ b/megatron/core/optimizer/__init__.py
@@ -53,6 +53,7 @@ def _get_param_groups(
     min_lr: float,
     decoupled_lr: Optional[float],
     decoupled_min_lr: Optional[float],
+    vision_ration,
 ) -> List[Dict]:
     """Create parameter groups for optimizer.
 
@@ -108,6 +109,8 @@ def _get_param_groups(
             else:
                 # Do not regularize biases and norm parameters.
                 no_wd = name.endswith(".bias") or len(param.shape) == 1
+                # NOTE(lizhiyu): hack for qwen2.5vl
+                # no_wd = name.endswith(".bias")
 
             if scale_lr_cond is not None:
                 scale_lr = scale_lr_cond(name, param)
@@ -130,8 +133,14 @@ def _get_param_groups(
                 param, 'is_embedding_or_output_parameter', False
             ):
                 is_decoupled_lr = True
+            
+            is_vision_model_param = False
+            if "vision_model" in name:
+                is_vision_model_param = True
+            else:
+                is_vision_model_param = False
 
-            key = (wd_mult, _lr_mult, is_expert_parallel, is_decoupled_lr)
+            key = (wd_mult, _lr_mult, is_expert_parallel, is_decoupled_lr, is_vision_model_param)
             if key not in params_map:
                 params_map[key] = []
             if (
@@ -143,7 +152,7 @@ def _get_param_groups(
                 params_map[key].append(param)
 
     param_groups = []
-    for (wd_mult, _lr_mult, is_expert_parallel, is_decoupled_lr), params in params_map.items():
+    for (wd_mult, _lr_mult, is_expert_parallel, is_decoupled_lr, is_vision_model_param), params in params_map.items():
         assert len(params) > 0
         param_group = {
             'params': params,
@@ -151,6 +160,7 @@ def _get_param_groups(
             'lr_mult': _lr_mult,
             'is_expert_parallel': is_expert_parallel,
             'is_decoupled_lr': is_decoupled_lr,
+            'is_vision_model_param': is_vision_model_param,
         }
         # Ensure param_group has required keys for matching when loading optimizer state
         # See MegatronOptimizer._filter_and_reorder_param_groups.
@@ -163,6 +173,7 @@ def _get_param_groups(
         min_lr=min_lr,
         decoupled_lr=decoupled_lr,
         decoupled_min_lr=decoupled_min_lr,
+        vision_ration=vision_ration,
     )
 
     return param_groups
@@ -174,6 +185,7 @@ def _update_min_and_max_lr_in_param_groups(
     min_lr: float,
     decoupled_lr: Optional[float],
     decoupled_min_lr: Optional[float],
+    vision_ration = 0.1,
 ) -> List[Dict]:
     """
     Updates `max_lr` and `min_lr` values in each parameter group, and returns new list.
@@ -202,7 +214,7 @@ def _update_min_and_max_lr_in_param_groups(
             param_group['max_lr'] = decoupled_lr
             param_group['min_lr'] = decoupled_min_lr
         else:
-            param_group['max_lr'] = lr
+            param_group['max_lr'] = lr if not param_group['is_vision_model_param'] else lr * vision_ration # NOTE(lizhiyu): change the ration here
             param_group['min_lr'] = min_lr
     return param_groups
 
@@ -247,6 +259,7 @@ def _get_param_groups_and_buffers(
         min_lr=config.min_lr,
         decoupled_lr=config.decoupled_lr,
         decoupled_min_lr=config.decoupled_min_lr,
+        vision_ration=config.vision_ration, # NOTE(lizhiyu): The vision ration is used to scale the learning rate for vision model parameters. Added by FlagScale.
     )
     param_groups = list(filter(filter_fn, param_groups))
     buffers = {}
@@ -486,7 +499,9 @@ def get_megatron_optimizer(
     else:
         all_dense_model_chunks = [model_chunks]
         overlap_param_gather_with_optimizer_step_flags = [False]
-    model_parallel_rank = mpu.get_model_parallel_group().rank()
+    mp_group = mpu.get_model_parallel_group()
+    mp_group = [mp_group] if not isinstance(mp_group, list) else mp_group
+    model_parallel_rank = mp_group[0].rank()
 
     if (
         mpu.get_data_parallel_group(with_context_parallel=True, partial_data_parallel=False).size()
@@ -597,7 +612,12 @@ def get_megatron_optimizer(
         buffer_name='expert_parallel_buffers',
     )
     if len(moe_param_groups) > 0:
-        model_parallel_rank = mpu.get_expert_tensor_model_pipeline_parallel_group().rank()
+        expert_mp_group = mpu.get_expert_tensor_model_pipeline_parallel_group()
+        if not isinstance(expert_mp_group, list):
+            model_parallel_rank = mpu.get_expert_tensor_model_pipeline_parallel_group().rank()
+        else:
+            model_parallel_rank = expert_mp_group[0].rank()
+
         # Pass Gloo process groups into optimizer only if needed.
         if use_gloo_process_groups:
             data_parallel_group_gloo = mpu.get_expert_data_parallel_group_gloo(
