diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java
index e1be80dc8a..0ad81d45ee 100644
--- a/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java
+++ b/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java
@@ -404,13 +404,8 @@ protected Expression serializeField(
       Expression fieldValue, Expression buffer, Descriptor descriptor) {
     TypeRef<?> typeRef = descriptor.getTypeRef();
     boolean nullable = descriptor.isNullable();
-
-    boolean useRefTracking;
-    if (needWriteRef(typeRef)) {
-      useRefTracking = descriptor.isTrackingRef();
-    } else {
-      useRefTracking = false;
-    }
+    // descriptor.isTrackingRef() already includes the needWriteRef check
+    boolean useRefTracking = descriptor.isTrackingRef();
 
     if (useRefTracking) {
       return new If(
@@ -1774,14 +1769,10 @@ protected Expression deserializeField(
       Expression buffer, Descriptor descriptor, Function<Expression, Expression> callback) {
     TypeRef<?> typeRef = descriptor.getTypeRef();
     boolean nullable = descriptor.isNullable();
-
+    // descriptor.isTrackingRef() already includes the needWriteRef check
+    boolean useRefTracking = descriptor.isTrackingRef();
+    // Check if type normally needs ref (for preserveRefId when ref tracking is disabled)
     boolean typeNeedsRef = needWriteRef(typeRef);
-    boolean useRefTracking;
-    if (needWriteRef(typeRef)) {
-      useRefTracking = descriptor.isTrackingRef();
-    } else {
-      useRefTracking = false;
-    }
 
     if (useRefTracking) {
       return readRef(buffer, callback, () -> deserializeForNotNullForField(buffer, typeRef, null));
diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/ClassDef.java b/java/fory-core/src/main/java/org/apache/fory/meta/ClassDef.java
index 4855a41a61..8ecb12d2c1 100644
--- a/java/fory-core/src/main/java/org/apache/fory/meta/ClassDef.java
+++ b/java/fory-core/src/main/java/org/apache/fory/meta/ClassDef.java
@@ -363,22 +363,25 @@ public List<Descriptor> getDescriptors(TypeResolver resolver, Class<?> cls) {
         }
 
         if (e.getKey() instanceof Field) {
-          boolean refTracking = resolver.getFory().trackingRef();
+          boolean globalRefTracking = resolver.getFory().trackingRef();
           ForyField foryField = desc.getForyField();
-          // update ref tracking if
-          // - global ref tracking is disabled but field is tracking ref (@ForyField#ref set)
-          // - global ref tracking is enabled but field is not tracking ref (@ForyField#ref not set)
-          boolean needsUpdate =
-              (refTracking && foryField == null && !desc.isTrackingRef())
-                  || (foryField != null && desc.isTrackingRef());
+          // Compute the final isTrackingRef value:
+          // 1. If global ref tracking is enabled and no @ForyField, use global setting
+          // 2. If @ForyField(ref=true) is set, use that (but can be overridden if global is off)
+          // 3. Additionally, check if the type actually supports ref tracking
+          boolean wantsRefTracking =
+              (globalRefTracking && foryField == null)
+                  || (foryField != null && desc.isTrackingRef() && globalRefTracking);
+          // Compute the final tracking: type must support refs AND user/global wants tracking
+          boolean finalTrackingRef = wantsRefTracking && resolver.needToWriteRef(desc.getTypeRef());
+          boolean needsUpdate = finalTrackingRef != desc.isTrackingRef();
 
           if (needsUpdate) {
             if (newDescriptors[0] == null) {
               newDescriptors[0] = new HashMap<>();
             }
-            boolean newTrackingRef = refTracking && foryField == null;
             Descriptor newDescriptor =
-                new DescriptorBuilder(desc).trackingRef(newTrackingRef).build();
+                new DescriptorBuilder(desc).trackingRef(finalTrackingRef).build();
 
             descriptorsMap.put(fullName, newDescriptor);
             desc = newDescriptor;
diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java
index bc2c0276df..a65282cdc0 100644
--- a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java
+++ b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java
@@ -1226,21 +1226,25 @@ public List<Descriptor> getFieldDescriptors(Class<?> clz, boolean searchParent)
             return;
           }
 
-          boolean shouldTrack = fory.trackingRef();
+          boolean globalRefTracking = fory.trackingRef();
           boolean hasForyField = descriptor.getForyField() != null;
-          // update ref tracking if
-          // - global ref tracking is enabled but field is not tracking ref (@ForyField#ref not set)
-          // - global ref tracking is disabled but field is tracking ref (@ForyField#ref set)
-          boolean needsUpdate =
-              (shouldTrack && !hasForyField)
-                  || (!shouldTrack && hasForyField && descriptor.isTrackingRef());
+          // Compute the final isTrackingRef value:
+          // 1. If global ref tracking is enabled and no @ForyField, use global setting
+          // 2. If @ForyField(ref=true) is set, use that (but can be overridden if global is off)
+          // 3. Additionally, check if the type actually supports ref tracking
+          boolean wantsRefTracking =
+              (globalRefTracking && !hasForyField)
+                  || (hasForyField && descriptor.isTrackingRef() && globalRefTracking);
+          // Compute the final tracking: type must support refs AND user/global wants tracking
+          boolean finalTrackingRef = wantsRefTracking && needToWriteRef(descriptor.getTypeRef());
+          boolean needsUpdate = finalTrackingRef != descriptor.isTrackingRef();
 
           if (needsUpdate) {
             if (newDescriptors[0] == null) {
               newDescriptors[0] = new HashMap<>();
             }
             Descriptor newDescriptor =
-                new DescriptorBuilder(descriptor).trackingRef(shouldTrack).build();
+                new DescriptorBuilder(descriptor).trackingRef(finalTrackingRef).build();
             result.add(newDescriptor);
             newDescriptors[0].put(member, newDescriptor);
           } else {
diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java
index 8ff29fcb4b..eedce31742 100644
--- a/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java
+++ b/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java
@@ -996,9 +996,8 @@ private InternalFieldInfo(Fory fory, Descriptor d, short classId) {
       this.fieldAccessor = d.getField() != null ? FieldAccessor.createAccessor(d.getField()) : null;
       fieldConverter = d.getFieldConverter();
       nullable = d.isNullable();
-      if (fory.trackingRef()) {
-        trackingRef = d.isTrackingRef();
-      }
+      // descriptor.isTrackingRef() already includes the needToWriteRef check
+      trackingRef = d.isTrackingRef();
     }
 
     @Override