This example will change the failed/duplicate entry message

add_filter( 'frm_global_failed_msg', 'change_failed_message' );
function change_failed_message( $message ) {
    $message = 'place new message here';
    return $message;
}

add_filter('frm_global_failed_msg', 'change_global_failed_msg_form', 10, 2);
function change_global_failed_msg_form( $message, $settings ) {
	$target_form_id = 852; // change  852 with the ID of your form
	if ( $target_form_id === (int) $settings->current_form ) {
		$message = 'Custom failed message for target form ID';
	}
	return $message;
}