diff --git a/plugins/virtualenvwrapper/virtualenvwrapper.plugin.zsh b/plugins/virtualenvwrapper/virtualenvwrapper.plugin.zsh
index c30216f51..d359e0c21 100644
--- a/plugins/virtualenvwrapper/virtualenvwrapper.plugin.zsh
+++ b/plugins/virtualenvwrapper/virtualenvwrapper.plugin.zsh
@@ -56,7 +56,13 @@ if [[ ! $DISABLE_VENV_CD -eq 1 ]]; then
       if [[ -n $CD_VIRTUAL_ENV && "$ENV_NAME" != "$CD_VIRTUAL_ENV" ]]; then
         # We've just left the repo, deactivate the environment
         # Note: this only happens if the virtualenv was activated automatically
-        deactivate && unset CD_VIRTUAL_ENV
+        if [[ -n "$VIRTUAL_ENV" ]]; then
+          # Only deactivate if VIRTUAL_ENV was set
+          # User may have deactivated manually or via another mechanism
+          deactivate
+        fi
+        # clean up regardless
+        unset CD_VIRTUAL_ENV
       fi
       if [[ "$ENV_NAME" != "" ]]; then
         # Activate the environment only if it is not already active