code to open dialog from form java script in mscrm 2011:

function opendialog() {

    // If on a create form then save and reload sot he dialog can open
    if(Xrm.Page.ui.getFormType() == 1)
    {
        Xrm.Page.data.entity.save(null);
        return;
    }

    // If sale appointment then trigger dialog
    if (Xrm.Page.getAttribute("new_issalesappointment").getValue() == true) {

        window.open("/" + Xrm.Page.context.getOrgUniqueName() + "/cs/dialog/rundialog.aspx?DialogId=%7b9D975EF8-74E0-4EE3-9D50-78FE7BAABF74%7d&EntityName=appointment&ObjectId=" + Xrm.Page.data.entity.getId());

        // Set as being displayed so it doesn't trigger again on load
        Xrm.Page.getAttribute("new_displayeddialog").setValue(true);

    }
}

function opendialogonload() {

    if (Xrm.Page.getAttribute("new_displayeddialog").getValue() == null
      || Xrm.Page.getAttribute("new_displayeddialog").getValue() == false) {
        opendialog();
    }