From http://xquartz.macosforge.org/trac/ticket/675:
launchctl unload -w /Library/LaunchAgents/org.macosforge.xquartz.startx.plist
sudo rm -rf /opt/X11* /Library/Launch*/org.macosforge.xquartz.* /Applications/Utilities/XQuartz.app /private/etc/*paths.d/*XQuartz
sudo pkgutil --forget org.macosforge.xquartz.pkg