I found a solution that seems to work with custom code to force woocommerce to look only at my custom rating_product key

/**
 * Force WooCommerce to use only the main 'rating' meta and ignore 'size_fit'
 */
add_filter('woocommerce_product_get_average_rating', 'custom_woocommerce_average_rating', 20, 2);

function custom_woocommerce_average_rating($average, $product) {
    // Get all comments/ratings for the product
    $args = array(
        'post_id' => $product->get_id(),
        'status' => 'approve',
        'meta_key' => 'rating_product', // Only get comments with your custom rating field
    );
    
    $comments = get_comments($args);
    
    if (!$comments) return $average;
    
    $total = 0;
    $count = 0;
    
    foreach ($comments as $comment) {
        $rating = get_comment_meta($comment->comment_ID, 'rating_product', true);
        if ($rating) {
            $total += (float) $rating;
            $count++;
        }
    }
    
    return ($count > 0) ? ($total / $count) : $average;
}