This validator will only print out the number of examples and recommend to the user to increase the number of examples if less than 100.
"""
MIN_EXAMPLES=100
optional_suggestion= (
""
iflen(df) >=MIN_EXAMPLES
else". In general, we recommend having at least a few hundred examples. We've found that performance tends to linearly increase for every doubling of the number of examples"
)
immediate_msg= (
f"\n- Your file contains {len(df)} prompt-completion pairs{optional_suggestion}"
warn_message+=f"\n WARNING: Some of the additional columns/keys contain `{ac}` in their name. These will be ignored, and the column/key `{ac}` will be used instead. This could also result from a duplicate column/key in the provided file."
immediate_msg=f"\n- The input file should contain exactly two columns/keys per row. Additional columns/keys present are: {additional_columns}{warn_message}"
immediate_msg=f"\n- There are {len(long_indexes)} examples that are very long. These are rows: {long_indexes}\nFor conditional generation, and for classification the examples shouldn't be longer than 2048 tokens."
optional_msg=f"Remove {len(long_indexes)} long examples"
defoptional_fn(x):
long_indexes_to_drop=get_long_indexes(x)
iflong_indexes!=long_indexes_to_drop:
sys.stdout.write(f"The indices of the long examples has changed as a result of a previously applied recommendation.\nThe {len(long_indexes_to_drop)} long examples to be dropped are now at the following indices: {long_indexes_to_drop}\n")
returnx.drop(long_indexes_to_drop)
returnRemediation(
name="long_examples",
immediate_msg=immediate_msg,
optional_msg=optional_msg,
optional_fn=optional_fn,
)
defcommon_prompt_suffix_validator(df):
"""
This validator will suggest to add a common suffix to the prompt if one doesn't already exist in case of classification or conditional generation.
"""
error_msg=None
immediate_msg=None
optional_msg=None
optional_fn=None
# Find a suffix which is not contained within the prompt otherwise
error_msg=f"All prompts are identical: `{common_suffix}`\nConsider leaving the prompts blank if you want to do open-ended generation, otherwise ensure prompts are different"
f"\n- All prompts end with suffix `{common_suffix_new_line_handled}`"
)
iflen(common_suffix) >10:
immediate_msg+=f". This suffix seems very long. Consider replacing with a shorter suffix, such as `{display_suggested_suffix}`"
if (
df.prompt.str[: -len(common_suffix)]
.str.contains(common_suffix, regex=False)
.any()
):
immediate_msg+=f"\n WARNING: Some of your prompts contain the suffix `{common_suffix}` more than once. We strongly suggest that you review your prompts and add a unique suffix"
else:
immediate_msg="\n- Your data does not contain a common separator at the end of your prompts. Having a separator string appended to the end of the prompt makes it clearer to the fine-tuned model where the completion should begin. See https://platform.openai.com/docs/guides/fine-tuning/preparing-your-dataset for more detail and examples. If you intend to do open-ended generation, then you should leave the prompts empty"
ifcommon_suffix=="":
optional_msg= (
f"Add a suffix separator `{display_suggested_suffix}` to all prompts"
)
defoptional_fn(x):
returnadd_suffix(x, suggested_suffix)
returnRemediation(
name="common_completion_suffix",
immediate_msg=immediate_msg,
optional_msg=optional_msg,
optional_fn=optional_fn,
error_msg=error_msg,
)
defcommon_prompt_prefix_validator(df):
"""
This validator will suggest to remove a common prefix from the prompt if a long one exist.
immediate_msg=f"\n- All prompts start with prefix `{common_prefix}`"
ifMAX_PREFIX_LEN<len(common_prefix):
immediate_msg+=". Fine-tuning doesn't require the instruction specifying the task, or a few-shot example scenario. Most of the time you should only add the input data into the prompt, and the desired output into the completion"
optional_msg=f"Remove prefix `{common_prefix}` from all prompts"
defoptional_fn(x):
returnremove_common_prefix(x, common_prefix)
returnRemediation(
name="common_prompt_prefix",
immediate_msg=immediate_msg,
optional_msg=optional_msg,
optional_fn=optional_fn,
)
defcommon_completion_prefix_validator(df):
"""
This validator will suggest to remove a common prefix from the completion if a long one exist.
immediate_msg=f"\n- All completions start with prefix `{common_prefix}`. Most of the time you should only add the output data into the completion, without any prefix"
optional_msg=f"Remove prefix `{common_prefix}` from all completions"
error_msg=f"All completions are identical: `{common_suffix}`\nEnsure completions are different, otherwise the model will just repeat `{common_suffix}`"
f"\n- All completions end with suffix `{common_suffix_new_line_handled}`"
)
iflen(common_suffix) >10:
immediate_msg+=f". This suffix seems very long. Consider replacing with a shorter suffix, such as `{display_suggested_suffix}`"
if (
df.completion.str[: -len(common_suffix)]
.str.contains(common_suffix, regex=False)
.any()
):
immediate_msg+=f"\n WARNING: Some of your completions contain the suffix `{common_suffix}` more than once. We suggest that you review your completions and add a unique ending"
else:
immediate_msg="\n- Your data does not contain a common ending at the end of your completions. Having a common ending string appended to the end of the completion makes it clearer to the fine-tuned model where the completion should end. See https://platform.openai.com/docs/guides/fine-tuning/preparing-your-dataset for more detail and examples."
ifcommon_suffix=="":
optional_msg= (
f"Add a suffix ending `{display_suggested_suffix}` to all completions"
)
defoptional_fn(x):
returnadd_suffix(x, suggested_suffix)
returnRemediation(
name="common_completion_suffix",
immediate_msg=immediate_msg,
optional_msg=optional_msg,
optional_fn=optional_fn,
error_msg=error_msg,
)
defcompletions_space_start_validator(df):
"""
This validator will suggest to add a space at the start of the completion if it doesn't already exist. This helps with tokenization.
immediate_msg="\n- The completion should start with a whitespace character (` `). This tends to produce better results due to the tokenization we use. See https://platform.openai.com/docs/guides/fine-tuning/preparing-your-dataset for more details"
optional_msg="Add a whitespace character to the beginning of the completion"
optional_fn=add_space_start
returnRemediation(
name="completion_space_start",
immediate_msg=immediate_msg,
optional_msg=optional_msg,
optional_fn=optional_fn,
)
deflower_case_validator(df, column):
"""
This validator will suggest to lowercase the column values, if more than a third of letters are uppercase.
immediate_msg=f"\n- More than a third of your `{column}` column/key is uppercase. Uppercase {column}s tends to perform worse than a mixture of case encountered in normal language. We recommend to lower case the data if that makes sense in your domain. See https://platform.openai.com/docs/guides/fine-tuning/preparing-your-dataset for more details",
optional_msg=f"Lowercase all your data in column/key `{column}`",
immediate_msg="\n- Based on your file extension, your file is formatted as an Excel file"
necessary_msg="Your format `XLSX` will be converted to `JSONL`"
xls=pd.ExcelFile(fname)
sheets=xls.sheet_names
iflen(sheets) >1:
immediate_msg+="\n- Your Excel file contains more than one sheet. Please either save as csv or ensure all data is present in the first sheet. WARNING: Reading only the first sheet..."
df=pd.read_excel(fname, dtype=str).fillna("")
eliffname.lower().endswith(".txt"):
immediate_msg= (
"\n- Based on your file extension, you provided a text file"
)
necessary_msg="Your format `TXT` will be converted to `JSONL`"
immediate_msg="\n- Your JSON file appears to be in a JSONL format. Your file will be converted to JSONL format"
necessary_msg="Your format `JSON` will be converted to `JSONL`"
else:
error_msg="Your file must have one of the following extensions: .CSV, .TSV, .XLSX, .TXT, .JSON or .JSONL"
if"."infname:
error_msg+=f" Your file `{fname}` ends with the extension `.{fname.split('.')[-1]}` which is not supported."
else:
error_msg+=f" Your file `{fname}` is missing a file extension."
except (ValueError, TypeError):
file_extension_str=fname.split(".")[-1].upper()
error_msg=f"Your file `{fname}` does not appear to be in valid {file_extension_str} format. Please ensure your file is formatted as a valid {file_extension_str} file."
else:
error_msg=f"File {fname} does not exist."
remediation=Remediation(
name="read_any_format",
necessary_msg=necessary_msg,
immediate_msg=immediate_msg,
error_msg=error_msg,
)
returndf, remediation
defformat_inferrer_validator(df):
"""
This validator will infer the likely fine-tuning format of the data, and display it to the user if it is classification.
It will also suggest to use ada and explain train/validation split benefits.
"""
ft_type=infer_task_type(df)
immediate_msg=None
ifft_type=="classification":
immediate_msg=f"\n- Based on your data it seems like you're trying to fine-tune a model for {ft_type}\n- For classification, we recommend you try one of the faster and cheaper models, such as `ada`\n- For classification, you can estimate the expected model performance by keeping a held out dataset, which is not used for training"
Estimate the time it'll take to fine-tune the dataset
"""
ft_format=infer_task_type(df)
expected_time=1.0
ifft_format=="classification":
num_examples=len(df)
expected_time=num_examples*1.44
else:
size=df.memory_usage(index=True).sum()
expected_time=size*0.0515
defformat_time(time):
iftime<60:
returnf"{round(time, 2)} seconds"
eliftime<3600:
returnf"{round(time/60, 2)} minutes"
eliftime<86400:
returnf"{round(time/3600, 2)} hours"
else:
returnf"{round(time/86400, 2)} days"
time_string=format_time(expected_time+140)
sys.stdout.write(
f"Once your model starts training, it'll approximately take {time_string} to train a `curie` model, and less for `ada` and `babbage`. Queue will approximately take half an hour per job ahead of you.\n"
This function will write out a dataframe to a file, if the user would like to proceed, and also offer a fine-tuning command with the newly created file.
For classification it will optionally ask the user if they would like to split the data into train/valid files, and modify the suggested command to include the valid set.
input_text="\n\nYour data will be written to a new JSONL file. Proceed [Y/n]: "
ifnotany_remediationsandnotsplit:
sys.stdout.write(
f'\nYou can use your file for fine-tuning:\n> openai api fine_tunes.create -t "{fname}"{additional_params}\n\nAfter you’ve fine-tuned a model, remember that your prompt has to end with the indicator string `{common_prompt_suffix_new_line_handled}` for the model to start generating completions, rather than continuing with the prompt.{optional_ending_string}\n'
# Add -v VALID_FILE if we split the file into train / valid
files_string= ("s"ifsplitelse"") +" to `"+ ("` and `".join(fnames))
valid_string=f' -v "{fnames[1]}"'ifsplitelse""
separator_reminder= (
""
iflen(common_prompt_suffix_new_line_handled) ==0
elsef"After you’ve fine-tuned a model, remember that your prompt has to end with the indicator string `{common_prompt_suffix_new_line_handled}` for the model to start generating completions, rather than continuing with the prompt."
)
sys.stdout.write(
f'\nWrote modified file{files_string}`\nFeel free to take a look!\n\nNow use that file when fine-tuning:\n> openai api fine_tunes.create -t "{fnames[0]}"{valid_string}{additional_params}\n\n{separator_reminder}{optional_ending_string}\n'
)
estimate_fine_tuning_time(df)
else:
sys.stdout.write("Aborting... did not write the file\n")
definfer_task_type(df):
"""
Infer the likely fine-tuning task type from the data
"""
CLASSIFICATION_THRESHOLD=3# min_average instances of each class