Update the link url in the Moss 2010 to call SP.UI.ModalDialog.showModalDialog

<script type="text/javascript">
var navBarHelpOverrideKey = "WSSEndUser";
$(document).ready(function () {
var span1 = $("span").filter(function () {
return  $(this).text() == "NewIdea" && $(this).parent()[0].tagName == "A";
})[0];

if(span1)
{
var a = span1.parentNode;
a.href = "http://moss/Lists/TestCustomList/NewForm.aspx?Source=http://moss";
a.onclick = openDialog;
}

});

function CallBackfromDialog(dialogResult)
{
alert(dialogResult);
};

function openDialog() {
var options = {
url: "http://moss/Lists/TestCustomList/NewForm.aspx?Source=http://moss",
width: 800,
height: 600,
title: "New Idea",
dialogReturnValueCallback: CallBackfromDialog,
};

SP.UI.ModalDialog.showModalDialog(options);
return false;
}
</script>