axolotl lora example (#10789)

* Add axolotl lora example
* Modify readme
* Add comments in yml

python/llm/example/GPU/LLM-Finetuning/axolotl/README.md

@@ -52,7 +52,25 @@ For more details, please refer [hfhuboffline](https://huggingface.co/docs/huggin
 
 ### 3. Finetune Llama-2-7B
 
-This example shows how to run [Alpaca QLoRA finetune on Llama-2](https://github.com/artidoro/qlora) directly on Intel GPU, based on [axolotl Llama-2 qlora example](https://github.com/OpenAccess-AI-Collective/axolotl/blob/v0.4.0/examples/llama-2/qlora.yml). Note that only Llama-2-7B QLora example is verified on Intel ARC 770 with 16GB memory.
+This example shows how to run [Alpaca LoRA training](https://github.com/tloen/alpaca-lora/tree/main) and [Alpaca QLoRA finetune](https://github.com/artidoro/qlora) directly on Intel GPU. Note that only Llama-2-7B LoRA and QLoRA examples are verified on Intel ARC 770 with 16GB memory.
+
+#### 3.1 Alpaca LoRA
+
+Based on [axolotl Llama-2 LoRA example](https://github.com/OpenAccess-AI-Collective/axolotl/blob/v0.4.0/examples/llama-2/lora.yml).
+
+```
+accelerate launch finetune.py lora.yml
+```
+
+In v0.4.0, you can also use `train.py` instead of `-m axolotl.cli.train` or `finetune.py`.
+
+```
+accelerate launch train.py lora.yml
+```
+
+#### 3.2 Alpaca QLoRA
+
+Based on [axolotl Llama-2 QLoRA example](https://github.com/OpenAccess-AI-Collective/axolotl/blob/v0.4.0/examples/llama-2/qlora.yml).
 
 Modify parameters in `qlora.yml` based on your requirements. Then, launch finetuning with the following command.
 
@@ -66,6 +84,8 @@ In v0.4.0, you can also use `train.py` instead of `-m axolotl.cli.train` or `fin
 accelerate launch train.py qlora.yml
 ```
 
+#### 3.3 Expected Output
+
 Output in console
 
 ```

python/llm/example/GPU/LLM-Finetuning/axolotl/lora.yml

@@ -0,0 +1,70 @@
+# This file is copied from https://github.com/OpenAccess-AI-Collective/axolotl/blob/v0.4.0/examples/llama-2/lora.yml
+base_model: NousResearch/Llama-2-7b-hf
+model_type: LlamaForCausalLM
+tokenizer_type: LlamaTokenizer
+
+load_in_8bit: false
+load_in_4bit: true
+strict: false
+
+datasets:
+  - path: mhenrichsen/alpaca_2k_test
+    type: alpaca
+dataset_prepared_path:
+val_set_size: 0.05
+output_dir: ./lora-out
+
+sequence_len: 4096
+sample_packing: true
+pad_to_sequence_len: true
+
+adapter: lora
+lora_model_dir:
+lora_r: 16
+lora_alpha: 16
+lora_dropout: 0.05
+lora_target_linear: true
+lora_fan_in_fan_out:
+
+wandb_project:
+wandb_entity:
+wandb_watch:
+wandb_name:
+wandb_log_model:
+
+gradient_accumulation_steps: 4
+micro_batch_size: 1
+num_epochs: 4
+# adamw_bnb_8bit will lead to OOM
+# optimizer: adamw_bnb_8bit
+optimizer: adamw_torch
+lr_scheduler: cosine
+learning_rate: 0.0002
+
+train_on_inputs: false
+group_by_length: false
+bf16: true
+fp16: false
+tf32: false
+
+gradient_checkpointing: true
+early_stopping_patience:
+resume_from_checkpoint:
+local_rank:
+logging_steps: 1
+xformers_attention:
+# flash_attention is not supported
+flash_attention: false
+s2_attention:
+
+warmup_steps: 10
+evals_per_epoch: 4
+eval_table_size:
+eval_max_new_tokens: 128
+saves_per_epoch: 1
+debug:
+deepspeed:
+weight_decay: 0.0
+fsdp:
+fsdp_config:
+special_tokens:

python/llm/example/GPU/LLM-Finetuning/axolotl/qlora.yml

@@ -38,6 +38,9 @@ wandb_log_model:
 gradient_accumulation_steps: 4
 micro_batch_size: 1
 num_epochs: 4
+# paged_adamw_32bit is not supported
+# due to bitsandbytes issue https://github.com/TimDettmers/bitsandbytes/issues/1180
+# optimizer: paged_adamw_32bit
 optimizer: adamw_torch
 lr_scheduler: cosine
 learning_rate: 0.0002
@@ -54,6 +57,7 @@ resume_from_checkpoint:
 local_rank:
 logging_steps: 1
 xformers_attention:
+# flash_attention is not supported
 flash_attention: false
 
 warmup_steps: 10