grpo_argument.json

{
    "train_datasets": "PKU-SafeRLHF/train",
    "eval_datasets": "PKU-SafeRLHF/test",
    "ptx_datasets": "alpaca",
    "actor_model_name_or_path": "PKU-Alignment/alpaca-7b-reproduced",
    "reward_model_name_or_path": "PKU-Alignment/beaver-7b-v1.0-reward",
    "output_dir": "checkpoints/llama-grpo",
    "logging_dir": "log",
    "max_length": 2048,
    "use_fusemt": 1,
    "use_flash_attention": 1,
    "max_dec_len": 1024,
    "min_dec_len": 1,
    "top_p": 0.8,
    "temperature": 1.0,
    "num_return_sequences": 1,
    "repetition_penalty": 1.0,
    "num_train_epochs": 1,
    "max_steps": 17,
    "update_iters": 1,
    "per_device_prompt_batch_size": 2,
    "per_device_train_batch_size": 2,
    "gradient_accumulation_steps": 1,
    "learning_rate": 2e-6,
    "min_learning_rate": 2e-7,
    "weight_decay": 0.01,
    "lr_scheduler_type": "cosine",
    "warmup_ratio": 0.03,
    "recompute": 1,
    "recompute_granularity": "full",
    "recompute_use_reentrant": 1,
    "critic_learning_rate": 2e-6,
    "critic_min_learning_rate": 2e-7,
    "critic_weight_decay": 0.01,
    "critic_lr_scheduler_type": "cosine",
    "critic_warmup_ratio": 0.03,
    "critic_recompute": 1,
    "critic_recompute_granularity": "full",
    "normalize_reward": 1,
    "normalize_advantage": 1,
    "kl_coeff": 0.02,
    "clip_range_ratio": 0.2,
    "clip_range_score": 10.0,
    "clip_range_value": 5.0,
    "ptx_coeff": 16.0,
    "logging_steps": 1,
    "logging_dir": "vdl_log",
    "evaluation_strategy": "no",
    "per_device_eval_batch_size": 16,
    "eval_steps": 10000,
    "save_strategy": "steps",
    "save_steps": 400,
    "save_total_limit": 5,
    "bf16": 1,
    "fp16": 0,
    "fp16_opt_level": "O2",
    "do_train": 1,
    "do_eval": 0,
    "disable_tqdm": 1,
    "sharding_parallel_degree": 1,
    "sharding": "stage1",
    "tensor_parallel_degree": 8,
    "tensor_parallel_output": 0,
    "pipeline_parallel_degree": 1,
    "pipeline_parallel_config": "disable_p2p_cache_shape",
    "sequence_parallel": 0,
    "max_grad_norm": 1.0,
    "adam_beta1": 0.9,
    "adam_beta2": 0.95,
    "dataloader_drop_last": 0,
    "eval_mode": "",
    "offload_level": "freeze_model optimizer train_model",
    "release_grads": 1,
    "seed": 23,
    "use_fused_head_and_loss_fn": 0,
    "fused_linear":1,
    "autotuner_benchmark": 0,
    "skip_profile_timer": 1,
    "use_rm_server": true,
    "reward_server": "http://10.174.146.80:8048",
    "rl_algorithm": "grpo"
}