ayo/ipex-llm
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

LLM: add device id to benchmark utils. (#10877)

Browse source
This commit is contained in:
Cengguang Zhang 2024-04-25 14:01:51 +08:00 committed by GitHub
parent 1ce8d7bcd9
commit cd369c2715
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 4 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
python/llm/dev/benchmark/benchmark_util.py
View file

@ -2443,7 +2443,7 @@ class BenchmarkWrapper:
if self.device.type == "xpu":
torch.xpu.synchronize()
memory_every_token.append(torch.xpu.memory.memory_reserved() / (1024**3))
memory_every_token.append(torch.xpu.memory.memory_reserved(self.device) / (1024**3))
self.peak_memory = np.max(memory_every_token)
end = time.perf_counter()
if first_token_time is None:
@ -2758,7 +2758,7 @@ class BenchmarkWrapper:
if self.device.type == "xpu":
torch.xpu.synchronize()
memory_every_token.append(torch.xpu.memory.memory_reserved() / (1024 ** 3))
memory_every_token.append(torch.xpu.memory.memory_reserved(self.device) / (1024 ** 3))
self.peak_memory = np.max(memory_every_token)
end = time.perf_counter()
if first_token_time is None:
@ -3099,7 +3099,7 @@ class BenchmarkWrapper:
if self.device.type == "xpu":
torch.xpu.synchronize()
memory_every_token.append(torch.xpu.memory.memory_reserved() / (1024 ** 3))
memory_every_token.append(torch.xpu.memory.memory_reserved(self.device) / (1024 ** 3))
self.peak_memory = np.max(memory_every_token)
end = time.perf_counter()
if first_token_time is None:
@ -3471,7 +3471,7 @@ class BenchmarkWrapper:
if self.device.type == "xpu":
torch.xpu.synchronize()
memory_every_token.append(torch.xpu.memory.memory_reserved() / (1024 ** 3))
memory_every_token.append(torch.xpu.memory.memory_reserved(self.device) / (1024 ** 3))
self.peak_memory = np.max(memory_every_token)
end = time.perf_counter()
if first_token_time is None: