math differences between Mono and Lua in the low bits

The report compares math results from Mono and Lua using an LSL script, especially vector rotation; it does not point to a source file or test. Start by locating the math and vector-operation implementations and existing VM parity tests, then reproduce the rotation case in both VMs. Done means the relevant results agree at the reported precision and the behavior is covered by a regression test.

Run this LSL script in both Mono and Lua VMs:

float LOG10_2_24 = 7.224719895935548;
float LOG10_2_23 = 6.923689900271567;

string printFloat(float f) {
    // higher precision than (string). accurate for most numbers between about 0.0001 and 2^31. innacurate outside that range
    float origFloat = f;
    integer i = (integer)f;
    string s = (string)i;
    if (f == i) return s;
    if (-1 < f && f < 0) s = "-0";
    s += ".";
    f = llFabs(f - i);
    integer leadingZeros = -(integer)llCeil(llLog10(f));
    for (i = 0; i < leadingZeros; i++) s += "0";
    // This was written as one expression, but it was split to force float32 conversion between all intermediate results under slua vm, for consistency with mono vm
    // s += (string)((integer)( f * llPow(10, llCeil(LOG10_2_23 - llLog10(f))) ));
    float val = llLog10(f);
    val = LOG10_2_23 - val;
    val = llPow(10, llCeil(val));
    val *= f;
    s += (string)((integer)(val));
    while (llGetSubString(s, -1, -1) == "0") {
        s = llGetSubString(s, 0, -2);
    }
    if (origFloat != (float)s) llOwnerSay("!! Inaccurate float: " + s + " !!");
    return s;
}

string printVector(vector v) {
    // higher precision than (string)
    return "<" + printFloat(v.x) + ", " + printFloat(v.y) + ", " + printFloat(v.z) + ">";
}

string printQuaternion(rotation q) {
    // higher precision than default
    return "<" + printFloat(q.x) + ", " + printFloat(q.y) + ", " + printFloat(q.z) + ", " + printFloat(q.s) + ">";
}

default {
    state_entry() {
        rotation q = <-1.426371, 0.787597, -1.999946, -0.307976>;
        vector v = <-1.426371, 0.787597, -1.999946>;
        vector v2 = <1, 2, 3>;
        float f = 0.57923982938;
        float f2 = 0.3392398693;
        float f3 = 11.8392398693;
        llOwnerSay("rotate vector: " + printVector(v * q));
        llOwnerSay("rotate quaternion: " + printQuaternion(q * q));
        llOwnerSay("dot product: " + printFloat(v * v2));
        llOwnerSay("cross product: " + printVector(v % v2));
        llOwnerSay("sin: " + printFloat(llSin(f)));
        llOwnerSay("cos: " + printFloat(llCos(f)));
        llOwnerSay("tan: " + printFloat(llTan(f)));
        llOwnerSay("asin: " + printFloat(llAsin(f2)));
        llOwnerSay("acos: " + printFloat(llAcos(f2)));
        llOwnerSay("atan: " + printFloat(llAtan2(f, f2)));
        llOwnerSay("log: " + printFloat(llLog(f3)));
        llOwnerSay("log10: " + printFloat(llLog10(f3)));
        llOwnerSay("sqrt: " + printFloat(llSqrt(f)));
        llOwnerSay("pow: " + printFloat(llPow(f, f2)));
        llOwnerSay("add: " + printFloat(f + f3));
        llOwnerSay("subtract: " + printFloat(f - f3));
        llOwnerSay("multiply: " + printFloat(f * f3));
        llOwnerSay("divide: " + printFloat(f / f3));
        llOwnerSay("normalize vector: " + printVector(llVecNorm(v)));
        llOwnerSay("magnitude vector: " + printFloat(llVecMag(v)));
        llOwnerSay("get pos: " + printVector(llGetPos()));
    }
}

Output:

Mono: 10:03 AM] Object: rotate vector: <-9.62725736, 5.31586742, -13.49858856>
Luau: [9:54 AM] Object: rotate vector: <-9.62725832, 5.315866, -13.49858664>
Mono: [6:13 AM] Object: rotate quaternion: <0.8785761, -0.48512196, 1.23187078, -6.5597782 >
Luau: [9:54 AM] Object: rotate quaternion: <0.8785761, -0.48512196, 1.23187078, -6.5597782>
\Mono: [6:13 AM] Object: dot product: -5.8510151
Luau: [9:54 AM] Object: dot product: -5.8510151
Mono: [6:13 AM] Object: cross product: <6.36268328, 2.2791667 , -3.64033888>
Luau: [9:54 AM] Object: cross product: <6.36268328, 2.2791667, -3.64033888>
Mono: [6:28 AM] Object: sin: 0.54738796
Luau: [9:54 AM] Object: sin: 0.54738796
Mono: [6:28 AM] Object: cos: 0.83687904
Luau: [9:54 AM] Object: cos: 0.83687904
Mono: [6:28 AM] Object: tan: 0.65408252
Luau: [9:54 AM] Object: tan: 0.65408252
Mono: [6:42 AM] Object: asin: 0.34610872
Luau: [9:54 AM] Object: asin: 0.34610872
Mono: [6:42 AM] Object: acos: 1.22468758
Luau: [9:54 AM] Object: acos: 1.22468758
Mono: [6:42 AM] Object: atan: 1.040984748
Luau: [9:54 AM] Object: atan: 1.040984748
Mono: [6:50 AM] Object: log: 2.47141932
Luau: [9:54 AM] Object: log: 2.47141932
Mono: [6:50 AM] Object: log10: 1.073323848
Luau: [9:54 AM] Object: log10: 1.073323848
Mono: [6:50 AM] Object: sqrt: 0.76107808
Luau: [9:54 AM] Object: sqrt: 0.76107808
Mono: [6:55 AM] Object: pow: 0.83090648
Luau: [9:54 AM] Object: pow: 0.83090648
Mono: [7:02 AM] Object: add: 12.41847992
Luau: [9:54 AM] Object: add: 12.41847992
Mono: [7:02 AM] Object: subtract: -11.26000022
Luau: [9:54 AM] Object: subtract: -11.26000022
Mono: [7:02 AM] Object: multiply: 6.8577595
Luau: [9:54 AM] Object: multiply: 6.8577595
Mono: [7:02 AM] Object: divide: 0.048925424
Luau: [9:54 AM] Object: divide: 0.048925424
Luau: [9:54 AM] Object: !! Inaccurate float: -0.77527616 !!
Mono: [8:24 AM] Object: normalize vector: <-0.5529306, 0.30531082, -0.77527616>
Luau: [9:54 AM] Object: normalize vector: <-0.5529306, 0.30531082, -0.77527616>
Mono: [8:30 AM] Object: magnitude vector: 2.57965636
Luau: [9:54 AM] Object: magnitude vector: 2.57965636
Mono: Mono: [6:13 AM] Object: rotate vector: <-9.62725736, 5.31586742, -13.49858856>
Luau: [9:54 AM] Object: rotate vector: <-9.62725832, 5.315866, -13.49858664>
Mono: [6:13 AM] Object: rotate quaternion: <0.8785761, -0.48512196, 1.23187078, -6.5597782 >
Luau: [9:54 AM] Object: rotate quaternion: <0.8785761, -0.48512196, 1.23187078, -6.5597782>
\Mono: [6:13 AM] Object: dot product: -5.8510151
Luau: [9:54 AM] Object: dot product: -5.8510151
Mono: [6:13 AM] Object: cross product: <6.36268328, 2.2791667 , -3.64033888>
Luau: [9:54 AM] Object: cross product: <6.36268328, 2.2791667, -3.64033888>
Mono: [6:28 AM] Object: sin: 0.54738796
Luau: [9:54 AM] Object: sin: 0.54738796
Mono: [6:28 AM] Object: cos: 0.83687904
Luau: [9:54 AM] Object: cos: 0.83687904
Mono: [6:28 AM] Object: tan: 0.65408252
Luau: [9:54 AM] Object: tan: 0.65408252
Mono: [6:42 AM] Object: asin: 0.34610872
Luau: [9:54 AM] Object: asin: 0.34610872
Mono: [6:42 AM] Object: acos: 1.22468758
Luau: [9:54 AM] Object: acos: 1.22468758
Mono: [6:42 AM] Object: atan: 1.040984748
Luau: [9:54 AM] Object: atan: 1.040984748
Mono: [6:50 AM] Object: log: 2.47141932
Luau: [9:54 AM] Object: log: 2.47141932
Mono: [6:50 AM] Object: log10: 1.073323848
Luau: [9:54 AM] Object: log10: 1.073323848
Mono: [6:50 AM] Object: sqrt: 0.76107808
Luau: [9:54 AM] Object: sqrt: 0.76107808
Mono: [6:55 AM] Object: pow: 0.83090648
Luau: [9:54 AM] Object: pow: 0.83090648
Mono: [7:02 AM] Object: add: 12.41847992
Luau: [9:54 AM] Object: add: 12.41847992
Mono: [7:02 AM] Object: subtract: -11.26000022
Luau: [9:54 AM] Object: subtract: -11.26000022
Mono: [7:02 AM] Object: multiply: 6.8577595
Luau: [9:54 AM] Object: multiply: 6.8577595
Mono: [7:02 AM] Object: divide: 0.048925424
Luau: [9:54 AM] Object: divide: 0.048925424
Luau: [9:54 AM] Object: !! Inaccurate float: -0.77527616 !!
Mono: [8:24 AM] Object: normalize vector: <-0.5529306, 0.30531082, -0.77527616>
Luau: [9:54 AM] Object: normalize vector: <-0.5529306, 0.30531082, -0.77527616>
Mono: [8:30 AM] Object: magnitude vector: 2.57965636
Luau: [9:54 AM] Object: magnitude vector: 2.57965636
Mono: [1:03 AM] Object: get pos: <63.21201706, 191.1995697, 22.26473236>
Luau: [9:54 AM] Object: get pos: <63.21201706, 191.1995697, 22.26473236>

Given that my "printFloat" function verifies its output against the (float) parser (probably accurate), and printed only one error, I think the problem in this script lies primarily with the actual math operations, and not bugs in printFloat (though I did just write printFloat today)

None of these issues are visible if I print with the lower precision (string) instead