[Qwen2] add max_pos_len in modeling_qwen2.py & remove max_pos_len in qwen2

Author: chuxiaoyi2023

models/Qwen2/share_cache_demo/README.md

@@ -25,7 +25,7 @@ cd build && cmake .. && make && cp *cpython* .. && cd ..
 
 ## 3. 运行python demo
 ```shell
-python pipeline.py --model_path encrypted.bmodel  --tokenizer_path ../support/token_config/ --devid 0 --generation_mode penalty_sample --lib_path build/libcipher.so --embedding_path embedding.bin
+python3 pipeline.py --model_path encrypted.bmodel  --tokenizer_path ../support/token_config/ --devid 0 --generation_mode penalty_sample --lib_path build/libcipher.so --embedding_path embedding.bin
 ```
 * io_alone_mode：当io_alone_mode=0，则正常prefill；当io_alone_mode=1，则使用kvcache复用方案
 * model_path_list：模型路径，当使用多个模型时，用逗号隔开
@@ -70,3 +70,9 @@ cp files/Qwen-7B-Chat/* your_torch_model
 ```shell
 model_tool --encrypt -model origin.bmodel -net_name block_0 -lib ./build/libcipher.so -o encrypted.bmodel
 ```
+
+### 减少日志打印
+* 如果想要减少类似`Can't find network name`这种日志打印，可以执行如下命令
+```shell
+export BMRT_LOG_VERSION=3
+```

models/Qwen2/share_cache_demo/chat.cpp

@@ -75,7 +75,7 @@ class Qwen {
                             size_t size);
 
   // tensors
-  void make_in_tensors();
+  void make_in_tensors(bool read_bmodel);
   void free_in_tensors();
 
   // sample
@@ -499,8 +499,8 @@ void Qwen::init_params() {
   }
 }
 
-void Qwen::make_in_tensors() {
-  if (inputs_pid.device_mem.u.device.device_addr > 0x100000000 && inputs_pid.device_mem.u.device.device_addr < 0x500000000){
+void Qwen::make_in_tensors(bool read_bmodel) {
+  if (!read_bmodel && inputs_pid.device_mem.u.device.device_addr > 0x100000000 && inputs_pid.device_mem.u.device.device_addr < 0x500000000){
     free_in_tensors();
   }
 
@@ -557,7 +557,7 @@ void Qwen::init(const std::vector<int> &devices, const std::string &model_path,
   init_params();
 
   // step4 : make in tensors
-  make_in_tensors();
+  make_in_tensors(read_bmodel);
 }
 
 void Qwen::free_in_tensors() {

models/Qwen2/share_cache_demo/export_onnx.py

@@ -92,6 +92,7 @@ def forward(self, hidden_states, position_ids, attention_mask):
             attention_mask=attention_mask,
             position_ids=position_ids,
             use_cache=True,
+            max_pos_len=args.max_pos_len
         )
         present_k, present_v = past_kv
         return hidden_states.float(), present_k.float(), present_v.float()
@@ -110,6 +111,7 @@ def forward(self, hidden_states, position_ids, attention_mask, past_k, past_v):
             position_ids=position_ids,
             attention_mask=attention_mask,
             use_cache=True,
+            max_pos_len=args.max_pos_len
         )
         present_k, present_v = past_kv
         return hidden_states.float(), present_k.float(), present_v.float()

models/Qwen2/share_cache_demo/files/Qwen2-7B-Instruct/config.json

@@ -23,6 +23,5 @@
   "transformers_version": "4.41.2",
   "use_cache": true,
   "use_sliding_window": false,
-  "vocab_size": 152064,
-  "max_pos_len": 8704
+  "vocab_size": 152064
 }

models/Qwen2/share_cache_demo/files/Qwen2-7B-Instruct/modeling_qwen2.py

@@ -239,7 +239,6 @@ def __init__(self, config: Qwen2Config, layer_idx: Optional[int] = None):
         self.rope_theta = config.rope_theta
         self.is_causal = True
         self.attention_dropout = config.attention_dropout
-        self.max_pos_len = config.max_pos_len
 
         if (self.head_dim * self.num_heads) != self.hidden_size:
             raise ValueError(
@@ -265,6 +264,7 @@ def forward(
         past_key_value: Optional[Cache] = None,
         output_attentions: bool = False,
         use_cache: bool = False,
+        max_pos_len: Optional[int] = 0,
     ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
         bsz, q_len, _ = hidden_states.size()
 
@@ -294,7 +294,7 @@ def forward(
                 )
             # kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx)
 
-        cos, sin = self.rotary_emb(value_states, seq_len=self.max_pos_len)
+        cos, sin = self.rotary_emb(value_states, seq_len=max_pos_len)
         # if past_key_value is not None:
         #   cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len-1)
         # else:
@@ -764,6 +764,7 @@ def forward(
         past_key_value: Optional[Tuple[torch.Tensor]] = None,
         output_attentions: Optional[bool] = False,
         use_cache: Optional[bool] = False,
+        max_pos_len: Optional[int] = 0,
     ) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]:
         """
         Args:
@@ -791,6 +792,7 @@ def forward(
             past_key_value=past_key_value,
             output_attentions=output_attentions,
             use_cache=use_cache,
+            max_pos_len=max_pos_len
         )
         hidden_states = residual + hidden_states
 

models/Qwen2/share_cache_demo/pipeline.py

@@ -310,10 +310,10 @@ def main(args):
         engine = Qwen(args)
 
         # 1. test one sample
-        engine.test_sample()
+        # engine.test_sample()
 
         # 2. test random
-        # engine.test_random()
+        engine.test_random()
 
         # 2. test c-eval
         # engine.test_ceval()