Merge pull request abetlen#453 from wu-qing-157/main

abetlen · web-flow · commit b8e0bed295e7 · 2023-07-08T02:31:52.000-04:00
Fix incorrect token_logprobs (due to indexing after sorting)
diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py
@@ -964,7 +964,7 @@ def _create_completion(
                                 )
                             ],
                             "text_offset": [text_offset],
-                            "token_logprobs": [sorted_logprobs[int(token)][0]],
+                            "token_logprobs": [current_logprobs[int(token)]],
                             "top_logprobs": [top_logprob],
                         }
                     returned_tokens += 1
@@ -1039,7 +1039,7 @@ def _create_completion(
                             self.detokenize([token]).decode("utf-8", errors="ignore")
                         ],
                         "text_offset": [text_offset],
-                        "token_logprobs": [sorted_logprobs[int(token)][0]],
+                        "token_logprobs": [current_logprobs[int(token)]],
                         "top_logprobs": [top_logprob],
                     }
 
@@ -1163,7 +1163,7 @@ def _create_completion(
                         zip(logprobs_token, range(len(logprobs_token))), reverse=True
                     )
                 )
-                token_logprobs.append(sorted_logprobs[int(token)][0])
+                token_logprobs.append(logprobs_token[int(token)])
                 top_logprob: Optional[Dict[str, float]] = {
                     self.detokenize([i]).decode("utf-8", errors="ignore"): logprob
                     for logprob, i in sorted_logprobs[:logprobs]

Original file line number	Diff line number	Diff line change
`@@ -964,7 +964,7 @@ def _create_completion(`
`964`	`964`	`)`
`965`	`965`	`],`
`966`	`966`	`"text_offset": [text_offset],`
`967`		`- "token_logprobs": [sorted_logprobs[int(token)][0]],`
	`967`	`+ "token_logprobs": [current_logprobs[int(token)]],`
`968`	`968`	`"top_logprobs": [top_logprob],`
`969`	`969`	`}`
`970`	`970`	`returned_tokens += 1`
`@@ -1039,7 +1039,7 @@ def _create_completion(`
`1039`	`1039`	`self.detokenize([token]).decode("utf-8", errors="ignore")`
`1040`	`1040`	`],`
`1041`	`1041`	`"text_offset": [text_offset],`
`1042`		`- "token_logprobs": [sorted_logprobs[int(token)][0]],`
	`1042`	`+ "token_logprobs": [current_logprobs[int(token)]],`
`1043`	`1043`	`"top_logprobs": [top_logprob],`
`1044`	`1044`	`}`
`1045`	`1045`
`@@ -1163,7 +1163,7 @@ def _create_completion(`
`1163`	`1163`	`zip(logprobs_token, range(len(logprobs_token))), reverse=True`
`1164`	`1164`	`)`
`1165`	`1165`	`)`
`1166`		`- token_logprobs.append(sorted_logprobs[int(token)][0])`
	`1166`	`+ token_logprobs.append(logprobs_token[int(token)])`
`1167`	`1167`	`top_logprob: Optional[Dict[str, float]] = {`
`1168`	`1168`	`self.detokenize([i]).decode("utf-8", errors="ignore"): logprob`
`1169`	`1169`	`for logprob, i in sorted_logprobs[:logprobs]`