diff --git a/python/llm/test/inference_gpu/test_transformers_api.py b/python/llm/test/inference_gpu/test_transformers_api.py index ceb6b78d..b8c5903b 100644 --- a/python/llm/test/inference_gpu/test_transformers_api.py +++ b/python/llm/test/inference_gpu/test_transformers_api.py @@ -15,7 +15,7 @@ # -import os, time +import os import pytest import tempfile @@ -102,160 +102,5 @@ def test_transformers_auto_model_for_speech_seq2seq_int4(): assert 'Mr. Quilter is the apostle of the middle classes and we are glad to welcome his gospel.' in transcription[0] -prompt = "Once upon a time, there existed a little girl who liked to have adventures. She wanted to go to places and meet new people, and have fun" - -# @pytest.mark.parametrize('Model, Tokenizer, model_path',[ -# (AutoModelForCausalLM, AutoTokenizer, os.environ.get('MPT_7B_ORIGIN_PATH')), -# (AutoModelForCausalLM, AutoTokenizer, os.environ.get('LLAMA2_7B_ORIGIN_PATH')) -# ]) -# def test_optimize_model(Model, Tokenizer, model_path): -# with torch.inference_mode(): -# tokenizer = Tokenizer.from_pretrained(model_path, trust_remote_code=True) -# input_ids = tokenizer.encode(prompt, return_tensors="pt").to(device) - -# model = Model.from_pretrained(model_path, -# load_in_4bit=True, -# optimize_model=False, -# trust_remote_code=True) -# model = model.to(device) -# logits_base_model = (model(input_ids)).logits -# model.to('cpu') # deallocate gpu memory - -# model = Model.from_pretrained(model_path, -# load_in_4bit=True, -# optimize_model=True, -# trust_remote_code=True) -# model = model.to(device) -# logits_optimized_model = (model(input_ids)).logits -# model.to('cpu') - -# tol = 1e-02 -# num_false = torch.isclose(logits_optimized_model, logits_base_model, rtol=tol, atol=tol)\ -# .flatten().tolist().count(False) -# percent_false = num_false / logits_optimized_model.numel() -# assert percent_false < 1e-02 - -class Test_Optimize_Gpu_Model: - def setup(self): - - self.layer_outputs = [] - self.pre_layer_outputs = [] - - def run_optimize_gpu_model(self, Model, Tokenizer, model_path, self_attn, layer_norm, lower_bound): - with torch.inference_mode(): - def forward_hook(module, input, output, layer_name): - self.layer_outputs.append(output) - - def pre_forward_hook(module, input, output, layer_name): - self.pre_layer_outputs.append(output) - - tokenizer = Tokenizer.from_pretrained(model_path, trust_remote_code=True) - input_ids = tokenizer.encode(prompt, return_tensors="pt").to(device) - - model = Model.from_pretrained(model_path, - load_in_4bit=True, - optimize_model=False, - trust_remote_code=True) - model = model.to(device) - - for layer_name, layer_module in model.named_modules(): - if layer_name == layer_norm: - layer_module.register_forward_hook( - lambda module, input, output, layer_name=layer_name: pre_forward_hook(module, input, - output, layer_name)) - if layer_name == self_attn: - layer_module.register_forward_hook( - lambda module, input, output, layer_name=layer_name: forward_hook(module, input, - output, layer_name)) - logits_base_model = (model(input_ids)).logits - # the list `layer_output` has only one element. - layer_tensor = self.layer_outputs.pop() - model.to('cpu') - - opt_model = Model.from_pretrained(model_path, - load_in_4bit=True, - optimize_model=True, - trust_remote_code=True) - opt_model = opt_model.to(device) - - - def replace_forward_hook(module, input, output, layer_name): - output = self.pre_layer_outputs[0] - return output - - for layer_name, layer_module in opt_model.named_modules(): - if layer_name == layer_norm: - layer_module.register_forward_hook( - lambda module, input, output, layer_name=layer_name: replace_forward_hook(module, input, - output, layer_name)) - if layer_name == self_attn: - layer_module.register_forward_hook( - lambda module, input, output, layer_name=layer_name: forward_hook(module, input, - output, layer_name)) - logits_optimized_model = (opt_model(input_ids)).logits - # the list `layer_output` has only one element. - opt_layer_tensor = self.layer_outputs[0] - opt_model.to('cpu') - - attn_output_diff = [] - for i, (t1, t2) in enumerate(zip(layer_tensor, opt_layer_tensor)): - if t1 is not None and t2 is not None: - if isinstance(t1, torch.Tensor) and isinstance(t2, torch.Tensor): - # 'attn_output' is of type torch.Tensor. - attn_output_diff.append(t1 - t2) - else: - # 'past_key_value'is of type tuple as default. - for i, (t3, t4) in enumerate(zip(t1, t2)): - if model.config.architectures[0] == "ChatGLMModel" and \ - hasattr(model.config, 'padded_vocab_size') and \ - model.config.padded_vocab_size == 65024: - # chatglm2's past_key_value is expanded 16x for some speedup. - # We need to narrow it here. - t4 = t4[:, :, 15:17, :] - attn_output_diff.append(t3 - t4) - - max_diff_tensor = [torch.max(item).item() for item in attn_output_diff] - print(max_diff_tensor) - assert all(max_diff <= lower_bound for max_diff in max_diff_tensor) - - - # def test_falcon_gpu_model(self): - - # Model = AutoModelForCausalLM - # Tokenizer = AutoTokenizer - # model_path = os.environ.get('FALCON_7B_ORIGIN_PATH') - # # currently only compare the output of the last self-attention layer. - # layer_norm = "transformer.h.31.input_layernorm" - # self_attn = "transformer.h.31.self_attention" - # lower_bound = 0 - - # self.run_optimize_gpu_model(Model, Tokenizer, model_path, self_attn, layer_norm, lower_bound) - - - # def test_llama_gpu_model(self): - - # Model = AutoModelForCausalLM - # Tokenizer = AutoTokenizer - # model_path = os.environ.get('LLAMA2_7B_ORIGIN_PATH') - # # currently only compare the output of the last self-attention layer. - # layer_norm = "model.layers.31.input_layernorm" - # self_attn = "model.layers.31.self_attn" - # lower_bound = 5e-2 - - # self.run_optimize_gpu_model(Model, Tokenizer, model_path, self_attn, layer_norm, lower_bound) - - # def test_chatglm2_gpu_model(self): - - # Model = AutoModel - # Tokenizer = AutoTokenizer - # model_path = os.environ.get('CHATGLM2_6B_ORIGIN_PATH') - # # currently only need to compare the output of one self-attention layer. - # layer_norm = "transformer.encoder.layers.27.input_layernorm" - # self_attn = "transformer.encoder.layers.27.self_attention" - # lower_bound = 1e-3 - - # self.run_optimize_gpu_model(Model, Tokenizer, model_path, self_attn, layer_norm, lower_bound) - - if __name__ == '__main__': pytest.main([__file__]) diff --git a/python/llm/test/inference_gpu/test_transformers_api_attention.py b/python/llm/test/inference_gpu/test_transformers_api_attention.py new file mode 100644 index 00000000..a0d1864b --- /dev/null +++ b/python/llm/test/inference_gpu/test_transformers_api_attention.py @@ -0,0 +1,157 @@ + +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +import pytest + +import torch +from bigdl.llm.transformers import AutoModelForCausalLM, AutoModel +from transformers import LlamaTokenizer, AutoTokenizer + +device = os.environ['DEVICE'] +print(f'Running on {device}') + +PROMPT = "Once upon a time, there existed a little girl who liked to have adventures. She wanted to go to places and meet new people, and have fun" +TEST_MODEL_LIST = [ + ("MPT-7B", AutoModelForCausalLM, AutoTokenizer, os.environ.get('MPT_7B_ORIGIN_PATH')), + ("Llama2-7B", AutoModelForCausalLM, LlamaTokenizer, os.environ.get('LLAMA2_7B_ORIGIN_PATH')), + ("Falcon-7B", AutoModelForCausalLM, AutoTokenizer, os.environ.get('FALCON_7B_ORIGIN_PATH')), + ("ChatGLM2-6B", AutoModel, AutoTokenizer, os.environ.get('CHATGLM2_6B_ORIGIN_PATH')), +] + +class Test_Optimize_Gpu_Model: + def setup_method(self): + + self.layer_outputs = [] + self.pre_layer_outputs = [] + + def run_optimize_gpu_model(self, Name, Model, Tokenizer, model_path, self_attn, layer_norm, lower_bound): + with torch.inference_mode(): + def forward_hook(module, input, output, layer_name): + self.layer_outputs.append(output) + + def pre_forward_hook(module, input, output, layer_name): + self.pre_layer_outputs.append(output) + + tokenizer = Tokenizer.from_pretrained(model_path, trust_remote_code=True) + input_ids = tokenizer.encode(PROMPT, return_tensors="pt").to(device) + + model = Model.from_pretrained(model_path, + load_in_4bit=True, + optimize_model=False, + trust_remote_code=True) + model = model.to(device) + for layer_name, layer_module in model.named_modules(): + if layer_name == layer_norm: + layer_module.register_forward_hook( + lambda module, input, output, layer_name=layer_name: pre_forward_hook(module, input, + output, layer_name)) + if layer_name == self_attn: + layer_module.register_forward_hook( + lambda module, input, output, layer_name=layer_name: forward_hook(module, input, + output, layer_name)) + logits_base_model = (model(input_ids)).logits + # the list `layer_output` has only one element. + layer_tensor = self.layer_outputs.pop() + model.to('cpu') + + opt_model = Model.from_pretrained(model_path, + load_in_4bit=True, + optimize_model=True, + trust_remote_code=True) + opt_model = opt_model.to(device) + + + def replace_forward_hook(module, input, output, layer_name): + output = self.pre_layer_outputs[0] + return output + + for layer_name, layer_module in opt_model.named_modules(): + if layer_name == layer_norm: + layer_module.register_forward_hook( + lambda module, input, output, layer_name=layer_name: replace_forward_hook(module, input, + output, layer_name)) + if layer_name == self_attn: + layer_module.register_forward_hook( + lambda module, input, output, layer_name=layer_name: forward_hook(module, input, + output, layer_name)) + logits_optimized_model = (opt_model(input_ids)).logits + # the list `layer_output` has only one element. + opt_layer_tensor = self.layer_outputs[0] + opt_model.to('cpu') + + attn_output_diff = [] + for i, (t1, t2) in enumerate(zip(layer_tensor, opt_layer_tensor)): + if t1 is not None and t2 is not None: + if isinstance(t1, torch.Tensor) and isinstance(t2, torch.Tensor): + # 'attn_output' is of type torch.Tensor. + attn_output_diff.append(t1 - t2) + else: + # 'past_key_value'is of type tuple as default. + for i, (t3, t4) in enumerate(zip(t1, t2)): + if model.config.architectures[0] == "ChatGLMModel" and \ + hasattr(model.config, 'padded_vocab_size') and \ + model.config.padded_vocab_size == 65024: + # chatglm2's past_key_value is expanded 16x for some speedup. + # We need to narrow it here. + t4 = t4[:, :, 15:17, :] + attn_output_diff.append(t3 - t4) + + max_diff_tensor = [torch.max(item).item() for item in attn_output_diff] + print(max_diff_tensor) + + assert all(max_diff <= lower_bound for max_diff in max_diff_tensor) + + @pytest.mark.parametrize('Name, Model, Tokenizer, model_path',TEST_MODEL_LIST) + def test_dynamic_functions(self, Name, Model, Tokenizer, model_path): + if Name == "MPT-7B": + self.MPT_7B_gpu_model(Name, Model, Tokenizer, model_path) + elif Name == "Llama2-7B": + self.Llama2_7B_gpu_model(Name, Model, Tokenizer, model_path) + elif Name == "Falcon-7B": + self.Falcon_7B_gpu_model(Name, Model, Tokenizer, model_path) + elif Name == "ChatGLM2-6B": + self.Chatglm2_gpu_model(Name, Model, Tokenizer, model_path) + + + def MPT_7B_gpu_model(self, Name, Model, Tokenizer, model_path): + # currently only need to compare the output of one self-attention layer. + layer_norm = "transformer.blocks.31.norm_1" + self_attn = "transformer.blocks.31.attn" + lower_bound = 0 + self.run_optimize_gpu_model(Name, Model, Tokenizer, model_path, self_attn, layer_norm, lower_bound) + + def Llama2_7B_gpu_model(self, Name, Model, Tokenizer, model_path): + # currently only compare the output of the last self-attention layer. + layer_norm = "model.layers.31.input_layernorm" + self_attn = "model.layers.31.self_attn" + lower_bound = 5e-2 + self.run_optimize_gpu_model(Name, Model, Tokenizer, model_path, self_attn, layer_norm, lower_bound) + + def Falcon_7B_gpu_model(self, Name, Model, Tokenizer, model_path): + # currently only compare the output of the last self-attention layer. + layer_norm = "transformer.h.31.input_layernorm" + self_attn = "transformer.h.31.self_attention" + lower_bound = 0 + self.run_optimize_gpu_model(Name, Model, Tokenizer, model_path, self_attn, layer_norm, lower_bound) + + def Chatglm2_gpu_model(self, Name, Model, Tokenizer, model_path): + # currently only need to compare the output of one self-attention layer. + layer_norm = "transformer.encoder.layers.27.input_layernorm" + self_attn = "transformer.encoder.layers.27.self_attention" + lower_bound = 5e-3 + self.run_optimize_gpu_model(Name, Model, Tokenizer, model_path, self_attn, layer_norm, lower_bound) \ No newline at end of file diff --git a/python/llm/test/inference_gpu/test_transformers_api_disable_xmx.py b/python/llm/test/inference_gpu/test_transformers_api_final_logits.py similarity index 73% rename from python/llm/test/inference_gpu/test_transformers_api_disable_xmx.py rename to python/llm/test/inference_gpu/test_transformers_api_final_logits.py index 6f895a6b..7ff2aa78 100644 --- a/python/llm/test/inference_gpu/test_transformers_api_disable_xmx.py +++ b/python/llm/test/inference_gpu/test_transformers_api_final_logits.py @@ -14,33 +14,38 @@ # limitations under the License. # + import os import pytest + import torch -from bigdl.llm.transformers import AutoModelForCausalLM -from transformers import AutoTokenizer +from bigdl.llm.transformers import AutoModelForCausalLM, AutoModel +from transformers import LlamaTokenizer, AutoTokenizer device = os.environ['DEVICE'] print(f'Running on {device}') -prompt = "Once upon a time, there existed a little girl who liked to have adventures. She wanted to go to places and meet new people, and have fun" +PROMPT = "Once upon a time, there existed a little girl who liked to have adventures. She wanted to go to places and meet new people, and have fun" +TEST_MODEL_LIST = [ + ("MPT-7B", AutoModelForCausalLM, AutoTokenizer, os.environ.get('MPT_7B_ORIGIN_PATH')), + ("Falcon-7B", AutoModelForCausalLM, AutoTokenizer, os.environ.get('FALCON_7B_ORIGIN_PATH')), +] -@pytest.mark.parametrize('Model, Tokenizer, model_path',[ - (AutoModelForCausalLM, AutoTokenizer, os.environ.get('MPT_7B_ORIGIN_PATH')), - # (AutoModelForCausalLM, AutoTokenizer, os.environ.get('LLAMA2_7B_ORIGIN_PATH')), - (AutoModelForCausalLM, AutoTokenizer, os.environ.get('FALCON_7B_ORIGIN_PATH')), - ]) -def test_optimize_model(Model, Tokenizer, model_path): + +@pytest.mark.parametrize('Name, Model, Tokenizer, model_path',TEST_MODEL_LIST) +def test_optimize_model(Name, Model, Tokenizer, model_path): with torch.inference_mode(): tokenizer = Tokenizer.from_pretrained(model_path, trust_remote_code=True) - input_ids = tokenizer.encode(prompt, return_tensors="pt").to(device) + input_ids = tokenizer.encode(PROMPT, return_tensors="pt").to(device) model = Model.from_pretrained(model_path, load_in_4bit=True, optimize_model=False, trust_remote_code=True) model = model.to(device) + logits_base_model = (model(input_ids)).logits + model.to('cpu') # deallocate gpu memory model = Model.from_pretrained(model_path, @@ -54,6 +59,11 @@ def test_optimize_model(Model, Tokenizer, model_path): tol = 1e-03 num_false = torch.isclose(logits_optimized_model, logits_base_model, rtol=tol, atol=tol)\ .flatten().tolist().count(False) + percent_false = num_false / logits_optimized_model.numel() + assert percent_false < 1e-02 - \ No newline at end of file + + +if __name__ == '__main__': + pytest.main([__file__]) diff --git a/python/llm/test/run-llm-inference-tests-gpu.sh b/python/llm/test/run-llm-inference-tests-gpu.sh index 59ba2a0a..6951ddc3 100644 --- a/python/llm/test/run-llm-inference-tests-gpu.sh +++ b/python/llm/test/run-llm-inference-tests-gpu.sh @@ -19,7 +19,8 @@ fi export OMP_NUM_THREADS=$THREAD_NUM pytest ${LLM_INFERENCE_TEST_DIR}/test_transformers_api.py -v -s export BIGDL_LLM_XMX_DISABLED=1 -pytest ${LLM_INFERENCE_TEST_DIR}/test_transformers_api_disable_xmx.py -v -s +pytest ${LLM_INFERENCE_TEST_DIR}/test_transformers_api_final_logits.py -v -s +pytest ${LLM_INFERENCE_TEST_DIR}/test_transformers_api_attention.py -v -s unset BIGDL_LLM_XMX_DISABLED now=$(date "+%s")