rayHit doesn't return the object but the intersection point,
to know if it hit an object, the object is passed in argument :
point = rayHit(start, end, object)
if point then
    -- there is a hit with object
end
Mikey shared a additional script function (written in c++) :
http://forum.maratis3d.com/viewtopic.php?id=434
Here is the function code :
int pickObject(void)
{
    MEngine * engine = MEngine::getInstance();
    MScriptContext * script = engine->getScriptContext();
    
    if (script->getArgsNumber() == 3)
    {
        MOCamera * camera = (MOCamera *)script->getPointer(0);
        float x = script->getFloat(1);
        float y = script->getFloat(2);
        MPhysicsContext * physics = engine->getPhysicsContext();
        
        unsigned int width, height;
        engine->getSystemContext()->getScreenSize(&width, &height);
        MVector3 rayO = camera->getTransformedPosition();
        MVector3 rayD = camera->getUnProjectedPoint(MVector3(x * float(width), (1 - y) * float(height), 1));
        rayD = rayO + ((rayD - rayO).getNormalized() * (camera->getClippingFar() - camera->getClippingNear()));
        unsigned int objId;
        if (physics->isRayHit(rayO, rayD, &objId))
        {
            MLevel * level = engine->getLevel();
            MScene * scene = level->getCurrentScene();
            unsigned int entCount = scene->getEntitiesNumber();
            for (unsigned int i = 0; i < entCount; ++i)
            {
                MOEntity * entity = scene->getEntityByIndex(i);
                MPhysicsProperties * phyProps = entity->getPhysicsProperties();
                if (phyProps)
                {
                    if (phyProps->getCollisionObjectId() == objId)
                    {
                        script->pushPointer(entity);
                        return 1;
                    }
                }
            }
        }
    }
    
    return 0;
}
Used like that in lua :
obj = pickObject(Camera, x, y)
if obj then
...
end
This is the way to add the function through a game plugin (to do in StartPlugin) :
MEngine* engine = MEngine::getInstance();
MScriptContext* script = engine->getScriptContext();
if(script)
    script->addFunction("pickObject", pickObject);