diff --git a/pom.xml b/pom.xml
index e7d1249728..24ace4f953 100644
--- a/pom.xml
+++ b/pom.xml
@@ -49,7 +49,7 @@
     <jakarta.ws.rs-api-version>3.1.0</jakarta.ws.rs-api-version>
     <snakeyaml.version>2.6</snakeyaml.version>
     <slf4j.version>2.0.17</slf4j.version>
-    <caffeine.version>2.9.3</caffeine.version>
+    <caffeine.version>3.0.0</caffeine.version>
     <protobuf.version>4.34.1</protobuf.version>
     <junit-jupiter.version>5.13.4</junit-jupiter.version>
     <junit-jupiter.version>5.13.4</junit-jupiter.version>
diff --git a/spring/src/main/java/io/kubernetes/client/spring/extended/manifests/KubernetesFromConfigMapProcessor.java b/spring/src/main/java/io/kubernetes/client/spring/extended/manifests/KubernetesFromConfigMapProcessor.java
index 142d7528c2..5312cd04d4 100644
--- a/spring/src/main/java/io/kubernetes/client/spring/extended/manifests/KubernetesFromConfigMapProcessor.java
+++ b/spring/src/main/java/io/kubernetes/client/spring/extended/manifests/KubernetesFromConfigMapProcessor.java
@@ -12,21 +12,18 @@
 */
 package io.kubernetes.client.spring.extended.manifests;
 
-import com.github.benmanes.caffeine.cache.CacheLoader;
+import com.github.benmanes.caffeine.cache.Cache;
 import com.github.benmanes.caffeine.cache.Caffeine;
-import com.github.benmanes.caffeine.cache.LoadingCache;
 import io.kubernetes.client.openapi.models.V1ConfigMap;
 import io.kubernetes.client.spring.extended.manifests.annotation.FromConfigMap;
 import io.kubernetes.client.spring.extended.manifests.config.KubernetesManifestsProperties;
 import io.kubernetes.client.spring.extended.manifests.configmaps.ConfigMapGetter;
 import java.lang.reflect.Field;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
-import java.util.function.Supplier;
-import org.checkerframework.checker.nullness.qual.NonNull;
-import org.checkerframework.checker.nullness.qual.Nullable;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.BeansException;
@@ -80,15 +77,7 @@ public Object postProcessAfterInitialization(Object bean, String beanName) throw
       ConfigMapGetter configMapGetter =
           getOrCreateConfigMapGetter(fromConfigMapAnnotation, applicationContext);
 
-      LoadingCache<String, String> configMapDataCache =
-          Caffeine.newBuilder()
-              .expireAfterWrite(manifestsProperties.getRefreshInterval())
-              .build(
-                  new ConfigMapGetterCacheLoader(
-                      () -> {
-                        return configMapGetter.get(
-                            fromConfigMapAnnotation.namespace(), fromConfigMapAnnotation.name());
-                      }));
+      Cache<String, String> configMapDataCache = Caffeine.newBuilder().build();
       fullyRefreshCache(configMapGetter, fromConfigMapAnnotation, configMapDataCache);
       configMapKeyRefresher.scheduleAtFixedRate(
           () -> {
@@ -106,14 +95,19 @@ public Object postProcessAfterInitialization(Object bean, String beanName) throw
   private static void fullyRefreshCache(
       ConfigMapGetter configMapGetter,
       FromConfigMap fromConfigMapAnnotation,
-      LoadingCache<String, String> configMapDataCache) {
+      Cache<String, String> configMapDataCache) {
     V1ConfigMap configMap =
         configMapGetter.get(fromConfigMapAnnotation.namespace(), fromConfigMapAnnotation.name());
     if (configMap == null || configMap.getData() == null) {
+      configMapDataCache.invalidateAll();
       return;
     }
     // TODO: make the cache data refreshment atomic
-    configMap.getData().keySet().stream().forEach(key -> configMapDataCache.refresh(key));
+    Map<String, String> newData = configMap.getData();
+    newData.forEach(configMapDataCache::put);
+    Set.copyOf(configMapDataCache.asMap().keySet()).stream()
+        .filter(key -> !newData.containsKey(key))
+        .forEach(configMapDataCache::invalidate);
   }
 
   private ConfigMapGetter getOrCreateConfigMapGetter(
@@ -144,22 +138,4 @@ private ConfigMapGetter getOrCreateConfigMapGetter(
   public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
     this.applicationContext = applicationContext;
   }
-
-  static class ConfigMapGetterCacheLoader implements CacheLoader<String, String> {
-
-    ConfigMapGetterCacheLoader(Supplier<V1ConfigMap> configMapSupplier) {
-      this.configMapSupplier = configMapSupplier;
-    }
-
-    private final Supplier<V1ConfigMap> configMapSupplier;
-
-    @Override
-    public @Nullable String load(@NonNull String key) throws Exception {
-      V1ConfigMap configMap = this.configMapSupplier.get();
-      if (configMap == null || configMap.getData() == null) {
-        return null;
-      }
-      return configMap.getData().get(key);
-    }
-  }
 }