From 233196eda1c313af5ae6598db812cdc4dfbed8af Mon Sep 17 00:00:00 2001
From: randaller <randaller@users.noreply.github.com>
Date: Sun, 19 Mar 2023 13:22:30 +0300
Subject: [PATCH] Update hf-inference-example.py

---
 hf-inference-example.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/hf-inference-example.py b/hf-inference-example.py
index 5ce2558..faacf47 100644
--- a/hf-inference-example.py
+++ b/hf-inference-example.py
@@ -1,5 +1,9 @@
 import llamahf
 
+# to save memory use bfloat16 on cpu
+# import torch
+# torch.set_default_dtype(torch.bfloat16)
+
 MODEL = 'decapoda-research/llama-7b-hf'
 # MODEL = 'decapoda-research/llama-13b-hf'
 # MODEL = 'decapoda-research/llama-30b-hf'