Refraction in ray tracer produces odd results, how do I combine all color components?

Question

I am writing a ray tracer, so far with only spheres, in C++ and after implementing Phong's reflection model, shadows and reflections, everything seemed to work fine. When I implemented refractions and fresnel I can't seem to get things to look right. I have been thinking whether or not it could be because of how I move the rayOrigin when I am inside/outside the sphere object but after trying and googling I still can't get it right.

Below is an image. The gray background is a large diffuse sphere and the smaller blue sphere behind the red sphere is also diffuse. The others are reflective and refractive with ior 1.5-1.6. There are two point lights, on slightly to left and one slighly to the right.

As seen in the image, the spheres don't appear transparent at all. There are also noticeable circular color differences on the spheres. Maybe this can be because of the way I combine the colors for each pixel in my trace function:

Vec3 trace(Vec3& rayOrigin, Vec3& rayDirection, unsigned recursiveDepth, std::vector<Sphere>& spheres, std::vector<Light>& lights, RenderOption& options) {
    //Finding nearest intersecting object
    float nearestDepth = 1e8;
    Sphere nearestObject;
    unsigned id = 0;
    Vec3 origin = rayOrigin + rayDirection * BIAS;
    for (unsigned i = 0; i < spheres.size(); ++i) {
        if (spheres[i].intersect(origin, rayDirection)) {
            if (spheres[i].depth < nearestDepth) {
                nearestDepth = spheres[i].depth;
                nearestObject = spheres[i];
                id = i;
            }
        }
    }

    Vec3 backgroundColor = Vec3(0.0f, 0.0f, 0.0f);
    if (!nearestObject.exists) {
        //No intersecting object -> background cooler
        return backgroundColor;
    } else {    
        Vec3 totalColor;
        Vec3 lightDirection;

        //Ambient color
        totalColor += options.ambientColor * nearestObject.ambientColor; //Ambient color set to 0

        //Calculate fresnel, update fresnelReflection & fresnelRefraction of nearestObject sent in
        fresnel(rayDirection, nearestObject);

        //Recursive reflection and refraction
        if ((nearestObject.reflectivity > 0.0f || nearestObject.transparency > 0.0f) && recursiveDepth < options.recursionDepth) {
            //Reflection case
            if (nearestObject.fresnelReflection > 0.0f) {
                Vec3 reflection = computeReflection(rayDirection, nearestObject.normal);
                Vec3 reflectedColor = trace(nearestObject.intersection, reflection, ++recursiveDepth, spheres, lights, options);
                totalColor += reflectedColor * nearestObject.fresnelReflection;
            }
            //Refraction case
            if (nearestObject.fresnelRefraction > 0.0f) {
                Vec3 refractionDirection = computeRefraction(rayDirection, nearestObject.normal, nearestObject.indexOfRefraction, nearestObject.intersection);
                Vec3 refractedColor = trace(nearestObject.intersection, refractionDirection, ++recursiveDepth, spheres, lights, options);
                totalColor += refractedColor * nearestObject.fresnelRefraction;
            }
        }
    
        //Phong reflection model and shadows
        for (unsigned i = 0; i < lights.size(); ++i) {
            //Shadow ray
            Vec3 intersectionPointBias = nearestObject.intersection + nearestObject.normal * BIAS;
            Vec3 shadowRayDirection = lights[i].position - intersectionPointBias; //normalized in intersect function

            for (unsigned k = 0; k < spheres.size(); ++k) //kolla inte nearestObject mot sig själv
            {
                if (!spheres[k].intersect(intersectionPointBias, shadowRayDirection))
                {
                    //Diffuse
                    lightDirection = lights[i].position - nearestObject.normal;
                    lightDirection.normalize();
                    totalColor += lights[i].diffuse * std::max(0.0f, nearestObject.normal.dot(lightDirection)) * nearestObject.diffuseColor;

                    //Specular
                    Vec3 viewDirection = nearestObject.intersection - options.cameraOrigin;
                    viewDirection.normalize();
                    Vec3 reflection = lightDirection - nearestObject.normal * 2 * (nearestObject.normal.dot(lightDirection));
                    reflection.normalize();
                    totalColor += lights[i].specular * nearestObject.specularColor * std::max(0.0f, pow(reflection.dot(viewDirection), nearestObject.shininessCoefficient));
                }
            }
        }
        return totalColor;
    }
}

Here are the other relevant functions: computeRefraction:

Vec3 computeRefraction(const Vec3& I, const Vec3& N, const float &ior, Vec3& intersection) {
    Vec3 normal = N; normal.normalize();
    normal = normal;
    Vec3 incident = I; incident.normalize();
    float cosi = incident.dot(normal);
    float n1, n2;
    if (cosi > 0.0f) { 
        //Incident and normal have same direction, INSIDE sphere
        n1 = ior;
        n2 = 1.0f;
        normal = -normal;
    } else { 
        //Incident and normal have opposite direction, OUTSIDE sphere
        n1 = 1.0f;
        n2 = ior;
        cosi = -cosi;
    }

    float eta = n1 / n2;
    float k = 1.0f - (eta * eta) * (1.0f - cosi * cosi);

    if (k < 0.0f) {   
        //internal reflection
        Vec3 reflectionRay = computeReflection(incident, normal);
        intersection = intersection + (normal * BIAS);
        return reflectionRay;
    } else {
        Vec3 refractionVector = incident * eta + normal * (eta * cosi - sqrt(k));
        refractionVector.normalize();
        intersection = intersection - (normal * BIAS);
        return refractionVector;
    }
}

fresnel:

void fresnel(const Vec3& I, Sphere& obj) {
    Vec3 normal = obj.normal;
    Vec3 incident = I;
    float cosi = clamp(-1.0f, 1.0f, incident.dot(normal));
    float etai = 1.0f, etat = obj.indexOfRefraction;
    if (cosi > 0) {
        std::swap(etai, etat);
    }
    float sint = etai / etat * sqrt(std::max(0.0f, 1 - cosi * cosi));
    if (sint >= 1) {
        obj.fresnelReflection = 1.0f;
        obj.fresnelRefraction = 0.0f;
    } else {
        float cost = sqrt(std::max(0.0f, 1 - sint * sint));
        cosi = abs(cost);
        float Rs = ((etat * cosi) - (etai * cost)) / ((etat * cosi) + (etai * cost));
        float Rp = ((etai * cosi) - (etat * cost)) / ((etai * cosi) + (etat * cost));
        obj.fresnelReflection = (Rs * Rs + Rp * Rp) / 2;
        obj.fresnelRefraction = 1.0f - obj.fresnelReflection;
    }
}

reflection:

Vec3 computeReflection(const Vec3& rayDirection, const Vec3& objectNormal){
    Vec3 normal = objectNormal;
    Vec3 incident = rayDirection;
    Vec3 reflection = incident - normal * (normal.dot(rayDirection)) * 2;
    reflection.normalize();
    return reflection;
}

Any help in understanding and resolving these rendering issues would be greatly appreciated as no other posts or theory has helped resolve this on my own this past week. Thank you!