diff --git a/packages/babel-core/src/config/caching.js b/packages/babel-core/src/config/caching.js
index 317e1e3a8a..219cf304d4 100644
--- a/packages/babel-core/src/config/caching.js
+++ b/packages/babel-core/src/config/caching.js
@@ -54,12 +54,14 @@ function makeCachedFunction<ArgT, ResultT, Cache: CacheMap<ArgT, ResultT>>(
       }
     }
 
-    const { cache, result } = makeCachePair();
+    const { cache, result, deactivate } = makeCacheConfig();
 
     const value = handler(arg, cache);
 
     if (autoPermacache && !result.configured) cache.forever();
 
+    deactivate();
+
     if (!result.configured) {
       // eslint-disable-next-line max-len
       throw new Error([
@@ -119,7 +121,7 @@ function makeCachedFunction<ArgT, ResultT, Cache: CacheMap<ArgT, ResultT>>(
   };
 }
 
-function makeCachePair(): { cache: CacheConfigurator, result: * } {
+function makeCacheConfig(): { cache: CacheConfigurator, result: *, deactivate: () => void } {
   const pairs = [];
 
   const result = {
@@ -130,6 +132,11 @@ function makeCachePair(): { cache: CacheConfigurator, result: * } {
     valid: () => pairs.every(([key, fn]) => key === fn()),
   };
 
+  let active = true;
+  const deactivate = () => {
+    active = false;
+  };
+
   const cache: CacheConfigurator = Object.assign((function cacheFn(val) {
     if (typeof val === "boolean") {
       if (val) cache.forever();
@@ -140,16 +147,19 @@ function makeCachePair(): { cache: CacheConfigurator, result: * } {
     return cache.using(val);
   }: any), ({
     forever() {
+      if (!active) throw new Error("Cannot change caching after evaluation has completed.");
       if (result.never) throw new Error("Caching has already been configured with .never()");
       result.forever = true;
       result.configured = true;
     },
     never() {
+      if (!active) throw new Error("Cannot change caching after evaluation has completed.");
       if (result.forever) throw new Error("Caching has already been configured with .forever()");
       result.never = true;
       result.configured = true;
     },
     using<T>(handler: () => T): T {
+      if (!active) throw new Error("Cannot change caching after evaluation has completed.");
       if (result.never || result.forever) {
         throw new Error("Caching has already been configured with .never or .forever()");
       }
@@ -160,6 +170,7 @@ function makeCachePair(): { cache: CacheConfigurator, result: * } {
       return key;
     },
     invalidate<T>(handler: () => T): T {
+      if (!active) throw new Error("Cannot change caching after evaluation has completed.");
       if (result.never || result.forever) {
         throw new Error("Caching has already been configured with .never or .forever()");
       }
@@ -172,5 +183,5 @@ function makeCachePair(): { cache: CacheConfigurator, result: * } {
     },
   }: CacheConfiguratorObj));
 
-  return { cache, result };
+  return { cache, result, deactivate };
 }