diff --git a/packages/babel-core/src/config/index.js b/packages/babel-core/src/config/index.js
index ef5d5c4a66..b62fc57fc4 100644
--- a/packages/babel-core/src/config/index.js
+++ b/packages/babel-core/src/config/index.js
@@ -3,7 +3,6 @@
 import path from "path";
 import * as context from "../index";
 import Plugin from "./plugin";
-import merge from "lodash/merge";
 import {
   buildRootChain,
   buildPresetChain,
@@ -13,10 +12,9 @@ import {
 } from "./config-chain";
 import type { UnloadedDescriptor } from "./config-descriptors";
 import traverse from "@babel/traverse";
-import clone from "lodash/clone";
 import { makeWeakCache, type CacheConfigurator } from "./caching";
 import { getEnv } from "./helpers/environment";
-import { validate } from "./validation/options";
+import { validate, type ValidatedOptions } from "./validation/options";
 import { validatePluginObject } from "./validation/plugins";
 
 type LoadedDescriptor = {
@@ -141,7 +139,8 @@ export default function loadConfig(inputOpts: mixed): ResolvedConfig | null {
     throw e;
   }
 
-  const opts: Object = merge(optionDefaults, options);
+  const opts: Object = optionDefaults;
+  merge(opts, options);
 
   // Tack the passes onto the object itself so that, if this object is passed back to Babel a second time,
   // it will be in the right structure to not change behavior.
@@ -161,6 +160,30 @@ export default function loadConfig(inputOpts: mixed): ResolvedConfig | null {
   };
 }
 
+function merge(target: ValidatedOptions, source: ValidatedOptions): void {
+  for (const k of Object.keys(source)) {
+    if (k === "parserOpts" && source.parserOpts) {
+      const parserOpts = source.parserOpts;
+      const targetObj = (target.parserOpts = target.parserOpts || {});
+      mergeObject(targetObj, parserOpts);
+    } else if (k === "generatorOpts" && source.generatorOpts) {
+      const generatorOpts = source.generatorOpts;
+      const targetObj = (target.generatorOpts = target.generatorOpts || {});
+      mergeObject(targetObj, generatorOpts);
+    } else {
+      const val = source[k];
+      if (val !== undefined) target[k] = (val: any);
+    }
+  }
+}
+
+function mergeObject<T: {}>(target: T, source: T) {
+  for (const k of Object.keys(source)) {
+    const val = source[k];
+    if (val !== undefined) target[k] = (val: any);
+  }
+}
+
 /**
  * Load a generic plugin/preset from the given descriptor loaded from the config object.
  */
@@ -238,7 +261,7 @@ const instantiatePlugin = makeWeakCache(
 
     const plugin = Object.assign({}, pluginObj);
     if (plugin.visitor) {
-      plugin.visitor = traverse.explode(clone(plugin.visitor));
+      plugin.visitor = traverse.explode(Object.assign({}, plugin.visitor));
     }
 
     if (plugin.inherits) {