From 7d3791c8195a3943ded38b2f5921da2ee894d12e Mon Sep 17 00:00:00 2001
From: Ziteng Zhang <87107332+Jasonzzt@users.noreply.github.com>
Date: Wed, 15 May 2024 09:17:32 +0800
Subject: [PATCH] [LLM] Add llama3 alpaca qlora example (#11011)

* Add llama3 finetune example based on alpaca qlora example
---
 .../QLoRA/alpaca-qlora/README.md              | 34 ++++++++++++++-----
 .../qlora_finetune_llama3_8b_arc_1_card.sh    | 21 ++++++++++++
 2 files changed, 47 insertions(+), 8 deletions(-)
 create mode 100644 python/llm/example/GPU/LLM-Finetuning/QLoRA/alpaca-qlora/qlora_finetune_llama3_8b_arc_1_card.sh

diff --git a/python/llm/example/GPU/LLM-Finetuning/QLoRA/alpaca-qlora/README.md b/python/llm/example/GPU/LLM-Finetuning/QLoRA/alpaca-qlora/README.md
index 10cdd1c6..4a4ffa9e 100644
--- a/python/llm/example/GPU/LLM-Finetuning/QLoRA/alpaca-qlora/README.md
+++ b/python/llm/example/GPU/LLM-Finetuning/QLoRA/alpaca-qlora/README.md
@@ -5,6 +5,7 @@ This example ports [Alpaca-LoRA](https://github.com/tloen/alpaca-lora/tree/main)
 > Note: You could also refer to [simple QLoRA example](../simple-example/) to try related usage.
 
 ### 0. Requirements
+
 To run this example with IPEX-LLM on Intel GPUs, we have some recommended requirements for your machine, please refer to [here](../../../README.md#requirements) for more information.
 
 ### 1. Install
@@ -26,6 +27,7 @@ pip install git+https://github.com/intel/intel-extension-for-deepspeed.git@ec332
 ```
 
 ### 2. Configures OneAPI environment variables
+
 ```bash
 source /opt/intel/oneapi/setvars.sh
 ```
@@ -129,8 +131,21 @@ bash qlora_finetune_llama2_70b_pvc_1550_4_card.sh
 
 </details>
 
+<details>
+  <summary> Show LLaMA3-8B examples </summary>
+
+##### Finetuning LLaMA3-8B on single Arc A770
+
+```bash
+bash qlora_finetune_llama3_8b_arc_1_card.sh
+```
+
+</details>
+
 ### 4. (Optional) Resume Training
+
 If you fail to complete the whole finetuning process, it is suggested to resume training from a previously saved checkpoint by specifying `resume_from_checkpoint` to the local checkpoint folder as following:**
+
 ```bash
 python ./alpaca_qlora_finetuning.py \
     --base_model "meta-llama/Llama-2-7b-hf" \
@@ -140,19 +155,21 @@ python ./alpaca_qlora_finetuning.py \
 ```
 
 ### 5. Sample Output
+
 ```log
-{'loss': 1.9231, 'learning_rate': 2.9999945367033285e-05, 'epoch': 0.0}                                                                                                                            
-{'loss': 1.8622, 'learning_rate': 2.9999781468531096e-05, 'epoch': 0.01}                                                                                                                           
-{'loss': 1.9043, 'learning_rate': 2.9999508305687345e-05, 'epoch': 0.01}                                                                                                                           
-{'loss': 1.8967, 'learning_rate': 2.999912588049185e-05, 'epoch': 0.01}                                                                                                                            
-{'loss': 1.9658, 'learning_rate': 2.9998634195730358e-05, 'epoch': 0.01}                                                                                                                           
-{'loss': 1.8386, 'learning_rate': 2.9998033254984483e-05, 'epoch': 0.02}                                                                                                                           
-{'loss': 1.809, 'learning_rate': 2.999732306263172e-05, 'epoch': 0.02}                                                                                                                             
-{'loss': 1.8552, 'learning_rate': 2.9996503623845395e-05, 'epoch': 0.02}                                                                                                                           
+{'loss': 1.9231, 'learning_rate': 2.9999945367033285e-05, 'epoch': 0.0}                                                                                                                          
+{'loss': 1.8622, 'learning_rate': 2.9999781468531096e-05, 'epoch': 0.01}                                                                                                                         
+{'loss': 1.9043, 'learning_rate': 2.9999508305687345e-05, 'epoch': 0.01}                                                                                                                         
+{'loss': 1.8967, 'learning_rate': 2.999912588049185e-05, 'epoch': 0.01}                                                                                                                          
+{'loss': 1.9658, 'learning_rate': 2.9998634195730358e-05, 'epoch': 0.01}                                                                                                                         
+{'loss': 1.8386, 'learning_rate': 2.9998033254984483e-05, 'epoch': 0.02}                                                                                                                         
+{'loss': 1.809, 'learning_rate': 2.999732306263172e-05, 'epoch': 0.02}                                                                                                                           
+{'loss': 1.8552, 'learning_rate': 2.9996503623845395e-05, 'epoch': 0.02}                                                                                                                         
   1%|█                                                                                                                                                         | 8/1164 [xx:xx<xx:xx:xx, xx s/it]
 ```
 
 ### 6. Merge the adapter into the original model
+
 ```
 python ./export_merged_model.py --repo-id-or-model-path REPO_ID_OR_MODEL_PATH --adapter_path ./outputs/checkpoint-200 --output_path ./outputs/checkpoint-200-merged
 ```
@@ -160,4 +177,5 @@ python ./export_merged_model.py --repo-id-or-model-path REPO_ID_OR_MODEL_PATH --
 Then you can use `./outputs/checkpoint-200-merged` as a normal huggingface transformer model to do inference.
 
 ### 7. Troubleshooting
+
 Please refer to [here](../../README.md#troubleshooting) for solutions of common issues during finetuning.
diff --git a/python/llm/example/GPU/LLM-Finetuning/QLoRA/alpaca-qlora/qlora_finetune_llama3_8b_arc_1_card.sh b/python/llm/example/GPU/LLM-Finetuning/QLoRA/alpaca-qlora/qlora_finetune_llama3_8b_arc_1_card.sh
new file mode 100644
index 00000000..dbf6d7b4
--- /dev/null
+++ b/python/llm/example/GPU/LLM-Finetuning/QLoRA/alpaca-qlora/qlora_finetune_llama3_8b_arc_1_card.sh
@@ -0,0 +1,21 @@
+
+# Copyright 2016 The BigDL Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# You could also specify `--base_model` to the local path of the huggingface model checkpoint folder and `--data_path` to the local path of the dataset JSON file
+python ./alpaca_qlora_finetuning.py \
+    --base_model "meta-llama/Meta-Llama-3-8B-Instruct" \
+    --data_path "yahma/alpaca-cleaned" \
+    --output_dir "./ipex-llm-qlora-alpaca"