diff --git a/python/llm/src/bigdl/llm/utils/lazy_load_torch.py b/python/llm/src/bigdl/llm/utils/lazy_load_torch.py
index eda2d6de..5de30b94 100644
--- a/python/llm/src/bigdl/llm/utils/lazy_load_torch.py
+++ b/python/llm/src/bigdl/llm/utils/lazy_load_torch.py
@@ -179,6 +179,7 @@ def lazyload(
                       f"but get {pickle_paths}. ")
     pickle_fp = zf.open(pickle_paths[0], 'r')
     state_dict = _load(pickle_fp, None, pickle, pickle_file=pickle_paths[0][:-4], zip_file=zf)
+    fp.close()  # Otherwise on windows this may be marked as reading
     return state_dict