diff --git a/src/video_core/texture_cache/image_base.cpp b/src/video_core/texture_cache/image_base.cpp
index 25a211df8..1909c9ecb 100644
--- a/src/video_core/texture_cache/image_base.cpp
+++ b/src/video_core/texture_cache/image_base.cpp
@@ -256,6 +256,8 @@ void AddImageAlias(ImageBase& lhs, ImageBase& rhs, ImageId lhs_id, ImageId rhs_i
     }
     lhs.aliased_images.push_back(std::move(lhs_alias));
     rhs.aliased_images.push_back(std::move(rhs_alias));
+    lhs.flags &= ~ImageFlagBits::IsRescalable;
+    rhs.flags &= ~ImageFlagBits::IsRescalable;
 }
 
 } // namespace VideoCommon
diff --git a/src/video_core/texture_cache/image_base.h b/src/video_core/texture_cache/image_base.h
index 9c34687e0..bab290ac7 100644
--- a/src/video_core/texture_cache/image_base.h
+++ b/src/video_core/texture_cache/image_base.h
@@ -36,8 +36,9 @@ enum class ImageFlagBits : u32 {
 
     // Rescaler
     Rescaled = 1 << 12,
-    RescaleChecked = 1 << 13,
-    Blacklisted = 1 << 14,
+    CheckingRescalable = 1 << 13,
+    IsRescalable = 1 << 14,
+    Blacklisted = 1 << 15,
 };
 DECLARE_ENUM_FLAG_OPERATORS(ImageFlagBits)
 
diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h
index c77332b46..c1fb12679 100644
--- a/src/video_core/texture_cache/texture_cache.h
+++ b/src/video_core/texture_cache/texture_cache.h
@@ -795,25 +795,25 @@ bool TextureCache<P>::BlackListImage(ImageId image_id) {
 
 template <class P>
 bool TextureCache<P>::ImageCanRescale(ImageBase& image) {
-    if (True(image.flags & ImageFlagBits::Blacklisted)) {
+    if (!image.info.rescaleable || True(image.flags & ImageFlagBits::Blacklisted)) {
         return false;
     }
-    if (True(image.flags & (ImageFlagBits::Rescaled | ImageFlagBits::RescaleChecked))) {
+    if (True(image.flags & (ImageFlagBits::Rescaled | ImageFlagBits::CheckingRescalable))) {
         return true;
     }
-    if (!image.info.rescaleable) {
-        image.flags &= ~ImageFlagBits::RescaleChecked;
-        return false;
+    if (True(image.flags & ImageFlagBits::IsRescalable)) {
+        return true;
     }
-    image.flags |= ImageFlagBits::RescaleChecked;
+    image.flags |= ImageFlagBits::CheckingRescalable;
     for (const auto& alias : image.aliased_images) {
         Image& other_image = slot_images[alias.id];
         if (!ImageCanRescale(other_image)) {
-            image.flags &= ~ImageFlagBits::RescaleChecked;
+            image.flags &= ~ImageFlagBits::CheckingRescalable;
             return false;
         }
     }
-    image.flags &= ~ImageFlagBits::RescaleChecked;
+    image.flags &= ~ImageFlagBits::CheckingRescalable;
+    image.flags |= ImageFlagBits::IsRescalable;
     return true;
 }