diff --git a/chromadb/segment/impl/manager/local.py b/chromadb/segment/impl/manager/local.py
index b4dd712fb40..23d2207c1b6 100644
--- a/chromadb/segment/impl/manager/local.py
+++ b/chromadb/segment/impl/manager/local.py
@@ -156,9 +156,9 @@ def prepare_segments_for_new_collection(
     def delete_segments(self, segments: Sequence[Segment]) -> Sequence[UUID]:
         for segment in segments:
             collection_id = segment["collection"]
-            self._vector_instances_file_handle_cache.evict(collection_id)
             if segment["id"] in self._instances:
                 if segment["type"] == SegmentType.HNSW_LOCAL_PERSISTED.value:
+                    self._vector_instances_file_handle_cache.evict(collection_id)
                     instance = self.get_segment(collection_id, VectorReader)
                     instance.delete()
                 elif segment["type"] == SegmentType.SQLITE.value: