Default Config
We use this default configuration to tag the results of metrics evaluations. Which you can override while calling the inspeq eval methods.
{
"global_communication_mode": "async",
"auto_instrument_mode": 1,
"evaluations": {
"LLM": {
"communication_mode": "sync",
"auto_instrument_mode": 1,
"configurations": {
"answer_relevance_config": {
"threshold": 0.5,
"custom_labels": [
"Irrelevant Answer",
"Marginally Relevant Answer",
"Relevant Answer"
],
"label_thresholds": [
0,
0.5,
0.7,
1
]
},
"answer_fluency_config": {
"threshold": 0.5,
"custom_labels": [
"PASS",
"FAIL"
],
"label_thresholds": [
0,
0.5,
1
]
},
"response_tone_config": {
"threshold": 0.5,
"custom_labels": [
"Negative",
"Neutral",
"Positive"
],
"label_thresholds": [
0,
0.5,
0.7,
1
]
},
"factual_consistency_config": {
"threshold": 0.5,
"custom_labels": [
"Hallucinated",
"Inconsistent",
"Consistent"
],
"label_thresholds": [
0,
0.5,
0.7,
1
]
},
"word_count_limit_config": {
"threshold": 0.5,
"custom_labels": [
"Failed",
"Passed"
],
"label_thresholds": [
0,
1
]
},
"do_not_use_keywords_config": {
"threshold": 0.5,
"custom_labels": [
"Failed",
"Passed"
],
"label_thresholds": [
0,
1
]
},
"coherence_config": {
"threshold": 0.75,
"custom_labels": [
"Coherent",
"Incoherent"
],
"label_thresholds": [
0,
1
]
},
"bias_config":{
"threshold": 0.5,
"custom_labels": [
"Unbiased",
"Biased"
],
"label_thresholds": [
0,
1
]
},
"grammatical_correctness_config": {
"threshold": 0.5,
"custom_labels": [
"Incorrect",
"Partially Correct",
"Correct"
],
"label_thresholds": [
0,
0.5,
0.7,
1
]
},
"conceptual_similarity_config": {
"threshold": 0.5,
"custom_labels": [
"Dissimilar",
"Somewhat similar",
"Similar"
],
"label_thresholds": [
0,
0.5,
0.7,
1
]
},
"readability_config": {
"threshold": 0.5,
"custom_labels": [
"Sophisticated",
"Moderate",
"Easy"
],
"label_thresholds": [
0,
0.3,
0.6,
1
]
},
"clarity_config": {
"threshold": 0.5,
"custom_labels": [
"Not Clear",
"Partially Clear",
"Clear"
],
"label_thresholds": [
0,
0.5,
0.74,
1
]
},
"narrative_continuity_config": {
"threshold": 0.5,
"custom_labels": [
"Non Continuous",
"Continuous"
],
"label_thresholds": [
0,
1
]
},
"model_refusal_config": {
"threshold": 0.5,
"custom_labels": [
"Detected",
"Not Detected"
],
"label_thresholds": [
0,
1
]
},
"data_leakage_config": {
"threshold": 0.5,
"custom_labels": [
"Not Detected",
"Detected"
],
"label_thresholds": [
0,
1
]
},
"creativity_config": {
"threshold": 0.5,
"custom_labels": [
"Not Creative",
"Creative"
],
"label_thresholds": [
0,
0.5,
1
]
},
"diversity_config": {
"threshold": 0.5,
"custom_labels": [
"Redundant",
"Non Redundant"
],
"label_thresholds": [
0,
0.41,
1
]
},
"insecure_output_config": {
"threshold": 0.5,
"custom_labels": [
"Not Detected",
"Detected"
],
"label_thresholds": [
0,
1
]
},
"prompt_injection_config": {
"threshold": 0.61,
"custom_labels": [
"Detected",
"Not Detected"
],
"label_thresholds": [
0,
0.61,
1
]
},
"compression_score_config": {
"threshold": 0.8,
"custom_labels": [
"Compact Summary",
"Loose Summary"
],
"label_thresholds": [
0,
1
]
},
"density_score_config": {
"threshold": 0.2,
"custom_labels": [
"Verbose Summarization",
"Concise Summarization"
],
"label_thresholds": [
0,
1
]
},
"fuzzy_score_config": {
"threshold": 0.8,
"custom_labels": [
"Well Aligned Summarization",
"Misaligned Summarization"
],
"label_thresholds": [
0,
1
]
},
"rouge_score_config": {
"threshold": 0.8,
"custom_labels": [
"High Overlap",
"Low Overlap"
],
"label_thresholds": [
0,
1
]
},
"bleu_score_config": {
"threshold": 0.8,
"custom_labels": [
"Highly Conforming",
"Poorly Conforming"
],
"label_thresholds": [
0,
1
]
},
"meteor_score_config": {
"threshold": 0.8,
"custom_labels": [
"Semantically Accurate",
"Semantically Drifting"
],
"label_thresholds": [
0,
1
]
},
"bert_score_config": {
"threshold": 0.8,
"custom_labels": [
"Linguistically Congruent",
"Linguistically Incongruent"
],
"label_thresholds": [
0,
1
]
},
"cosine_similarity_score_config": {
"threshold": 0.2,
"custom_labels": [
"Contextual Synchrony",
"Contextual Divergence"
],
"label_thresholds": [
0,
1
]
},
"euclidean_distance_score_config": {
"threshold": 0.5,
"custom_labels": [
"Detected",
"Not Detected"
],
"label_thresholds": [
0,
1
]
},
"toxicity_config":{
"threshold":0.5,
"custom_labels": [
"Not Detected",
"Detected"
]
},
"invisible_text_config": {
"threshold": 0.5,
"custom_labels": [
"Not Detected",
"Detected"
],
"label_thresholds": [
0,
1
]
},
"get_sentence_embedding_score_config": {
"threshold": 0.5,
"custom_labels": [
"Detected",
"Not Detected"
],
"label_thresholds": [
0,
1
]
}
}
},
"RAGS": {
"communication_mode": "sync",
"auto_instrument_mode": 1,
"configurations": {
"retrieval_relevance_config": {
"threshold": 0.5,
"custom_labels": [
"relevant",
"irrelevant"
],
"label_thresholds": [
0,
1
]
},
"response_coherence_config": {
"threshold": 0.5,
"custom_labels": [
"coherent",
"incoherent"
],
"label_thresholds": [
0
]
}
}
},
"SQL": {
"communication_mode": "async",
"auto_instrument_mode": 1,
"configurations": {
"query_performance_config": {
"threshold": 0.5,
"custom_labels": [
"efficient",
"inefficient"
],
"label_thresholds": [
0,
1
]
},
"data_security_config": {
"threshold": 0.5,
"custom_labels": [
"secure",
"insecure"
],
"label_thresholds": [
0,
1
]
}
}
}
}
}Last updated