diff --git a/unsloth/chat_templates.py b/unsloth/chat_templates.py index 07e79b180b..7070524e0f 100644 --- a/unsloth/chat_templates.py +++ b/unsloth/chat_templates.py @@ -1458,9 +1458,10 @@ extra_eos_tokens = None, ollama_eos = '\n'.join(f'PARAMETER stop "{eos}"' for eos in ollama_eos) # Ollama modelfile + part = '"""' modelfile = 'FROM {__FILE_LOCATION__}\n\n'\ - 'TEMPLATE """' + system_modelfile + input_modelfile + output_modelfile + \ - '"""\n\n' + ollama_eos + 'TEMPLATE ' + part + system_modelfile + input_modelfile + output_modelfile + \ + part + '\n\n' + ollama_eos # HF Jinja Chat template def process(part, which, content = "message['content']"): @@ -1659,6 +1660,70 @@ extra_eos_tokens = None, pass +# From https://www.geeksforgeeks.org/longest-common-substring-array-strings/ +# Longest Common Substring in an Array of Strings +def _longest_common_substring(arr): + n = len(arr) + s = arr[0] + l = len(s) + res = "" + for i in range(l): + for j in range(i + 1, l + 1): + stem = s[i:j] + k = 1 + for k in range(1, n): + if stem not in arr[k]: + break + if (k + 1 == n and len(res) < len(stem)): + res = stem + return res +pass + + +def _find_common_token_ids(component, tokenizer): + """ + \n### User:\n\n + \n\n### User:\n\n + etc + we need to find the middle most repeatted part. + Tokenizers can tokenize newlines or spaces as 1 token! + """ + right_text = "" + if component.endswith (" "): right_text = " " + elif component.endswith("\n"): right_text = "\n" + left_text = "" + if component.startswith (" "): left_text = " " + elif component.startswith("\n"): left_text = "\n" + stripped = component.strip() + + # Add current pieces and also newlines + all_input_ids = [] + for left in range(3): + for right in range(3): + x = left*left_text + stripped + right*right_text + x = tokenizer(x, add_special_tokens = False).input_ids + all_input_ids.append(x) + + x = left*"\n" + stripped + right*"\n" + x = tokenizer(x, add_special_tokens = False).input_ids + all_input_ids.append(x) + pass + pass + substring = _longest_common_substring([str(x + [0]) for x in all_input_ids]) + substring = substring.split(", ")[:-1] + substring = [int(x) for x in substring] + + # Also get rest of tokenized string + original = tokenizer(component, add_special_tokens = False).input_ids + # Get optional left and right + for j in range(len(original)): + if original[j : j + len(substring)] == substring: break + optional_left = original[:j] + optional_right = original[j+len(substring):] + return substring, optional_left, optional_right +pass + + def train_on_responses_only( trainer, instruction_part = None, @@ -1685,41 +1750,87 @@ def train_on_responses_only( response_part = tokenizer._unsloth_output_part pass - instruction_ids = tokenizer(instruction_part, add_special_tokens = False).input_ids - response_ids = tokenizer(response_part, add_special_tokens = False).input_ids + # Get most common tokens since tokenizers can tokenize stuff differently! + Q_must, Q_left, Q_right = _find_common_token_ids(instruction_part, tokenizer) + A_must, A_left, A_right = _find_common_token_ids(response_part, tokenizer) - instruction_length = len(instruction_ids) - response_length = len(response_ids) - max_length = max(instruction_length, response_length) + # Store some temporary stuff + A_first = A_must[0] + len_A_must = len(A_must) + A_left_reversed = A_left[::-1] + A_right_forward = A_right + + Q_first = Q_must[0] + len_Q_must = len(Q_must) + Q_left_reversed = Q_left[::-1] + Q_right_forward = Q_right def _train_on_responses_only(examples): input_ids_ = examples["input_ids"] all_labels = [] for input_ids in input_ids_: - - labels = [-100] * len(input_ids) - m = len(input_ids) - max_length - first_response = response_ids[0] - first_instruction = instruction_ids[0] + n = len(input_ids) + labels = [-100] * n + n_minus_1 = n - 1 j = 0 - while j < m: - if input_ids[j] == first_response: - if input_ids[j : j+response_length] == response_ids: - j = j + response_length - start = j - while j < m: - if input_ids[j] == first_instruction and input_ids[j : j+instruction_length] == instruction_ids: - j = j + instruction_length - labels[start : j] = input_ids[start : j] - break - elif j == (m-1): - j = m - labels[start:] = input_ids[start:] - break + while j < n: + # Find + if (input_ids[j] == A_first) and \ + (input_ids[j : (k := j + len_A_must)] == A_must): + + # Now backtrack to get previous optional tokens + for optional_left in A_left_reversed: + if j < 1: break + if optional_left == input_ids[j-1]: j -= 1 + else: break + pass + # And forwards look as well + for optional_right in A_right_forward: + if k >= n_minus_1: break + if optional_right == input_ids[k+1]: k += 1 + else: break + pass + # assistant_j = j + assistant_k = k + + j = assistant_k + # Given , now find next user + while j < n: + # Find + # Also accept last final item if assistant is the last turn + if (j == n_minus_1) or \ + ((input_ids[j] == Q_first) and \ + (input_ids[j : (k := j + len_Q_must)] == Q_must)): + + # Now backtrack to get previous optional tokens + for optional_left in Q_left_reversed: + if j < 1: break + if optional_left == input_ids[j-1]: j -= 1 + else: break pass - j += 1 + # And forwards look as well + for optional_right in Q_right_forward: + if k >= n_minus_1: break + if optional_right == input_ids[k+1]: k += 1 + else: break + pass + user_j = j + # Account for last item + if user_j != n_minus_1: + # user_k = k + # j = user_k + j = k + else: + user_j = n + k = n + pass + # Now copy input_ids to labels + labels[assistant_k : user_j] = input_ids[assistant_k : user_j] + # print(assistant_j, assistant_k, user_j, user_k) + break pass + j += 1 pass pass j += 1