Merge branch 'upstream-master'

Author: Datadog Syncup Service

src/hotspot/cpu/arm/stubDeclarations_arm.hpp

@@ -30,7 +30,13 @@
                                           do_arch_blob,                 \
                                           do_arch_entry,                \
                                           do_arch_entry_init)           \
-  do_arch_blob(preuniverse, 0)                                          \
+  do_arch_blob(preuniverse, 500)                                        \
+  do_stub(preuniverse, atomic_load_long)                                \
+  do_arch_entry(Arm, preuniverse, atomic_load_long,                     \
+                atomic_load_long_entry, atomic_load_long_entry)         \
+  do_stub(preuniverse, atomic_store_long)                               \
+  do_arch_entry(Arm, preuniverse, atomic_store_long,                    \
+                atomic_store_long_entry, atomic_store_long_entry)       \
 
 
 #define STUBGEN_INITIAL_BLOBS_ARCH_DO(do_stub,                          \
@@ -41,12 +47,6 @@
   do_stub(initial, idiv_irem)                                           \
   do_arch_entry(Arm, initial, idiv_irem,                                \
                 idiv_irem_entry, idiv_irem_entry)                       \
-  do_stub(initial, atomic_load_long)                                    \
-  do_arch_entry(Arm, initial, atomic_load_long,                         \
-                atomic_load_long_entry, atomic_load_long_entry)         \
-  do_stub(initial, atomic_store_long)                                   \
-  do_arch_entry(Arm, initial, atomic_store_long,                        \
-                atomic_store_long_entry, atomic_store_long_entry)       \
 
 #define STUBGEN_CONTINUATION_BLOBS_ARCH_DO(do_stub,                     \
                                            do_arch_blob,                \

src/hotspot/cpu/arm/stubGenerator_arm.cpp

@@ -3127,7 +3127,14 @@ class StubGenerator: public StubCodeGenerator {
   // Initialization
 
   void generate_preuniverse_stubs() {
-    // preuniverse stubs are not needed for arm
+    // Atomics are used in universe initialization code (e.g. CDS relocation),
+    // therefore we need to generate real stubs very early on.
+    StubRoutines::_atomic_add_entry = generate_atomic_add();
+    StubRoutines::_atomic_xchg_entry = generate_atomic_xchg();
+    StubRoutines::_atomic_cmpxchg_entry = generate_atomic_cmpxchg();
+    StubRoutines::_atomic_cmpxchg_long_entry = generate_atomic_cmpxchg_long();
+    StubRoutines::Arm::_atomic_load_long_entry = generate_atomic_load_long();
+    StubRoutines::Arm::_atomic_store_long_entry = generate_atomic_store_long();
   }
 
   void generate_initial_stubs() {
@@ -3151,14 +3158,6 @@ class StubGenerator: public StubCodeGenerator {
 
     // integer division used both by interpreter and compiler
     StubRoutines::Arm::_idiv_irem_entry = generate_idiv_irem();
-
-    StubRoutines::_atomic_add_entry = generate_atomic_add();
-    StubRoutines::_atomic_xchg_entry = generate_atomic_xchg();
-    StubRoutines::_atomic_cmpxchg_entry = generate_atomic_cmpxchg();
-    StubRoutines::_atomic_cmpxchg_long_entry = generate_atomic_cmpxchg_long();
-    StubRoutines::Arm::_atomic_load_long_entry = generate_atomic_load_long();
-    StubRoutines::Arm::_atomic_store_long_entry = generate_atomic_store_long();
-
   }
 
   void generate_continuation_stubs() {

src/hotspot/cpu/riscv/nativeInst_riscv.cpp

@@ -60,13 +60,13 @@ class NativeFarCall: public NativeInstruction {
   address next_instruction_address() const  { return addr_at(return_address_offset); }
   address return_address() const            { return addr_at(return_address_offset); }
   address destination() const;
-  address reloc_destination(address orig_address);
+  address reloc_destination();
 
   void set_destination(address dest);
   void verify();
   void print();
 
-  bool set_destination_mt_safe(address dest, bool assert_lock = true);
+  bool set_destination_mt_safe(address dest);
   bool reloc_set_destination(address dest);
 
  private:
@@ -88,28 +88,30 @@ address NativeFarCall::destination() const {
   address destination = MacroAssembler::target_addr_for_insn(addr);
 
   CodeBlob* cb = CodeCache::find_blob(addr);
-  assert(cb && cb->is_nmethod(), "sanity");
+  assert(cb != nullptr && cb->is_nmethod(), "nmethod expected");
   nmethod *nm = (nmethod *)cb;
   assert(nm != nullptr, "Sanity");
   assert(nm->stub_contains(destination), "Sanity");
   assert(destination != nullptr, "Sanity");
   return stub_address_destination_at(destination);
 }
 
-address NativeFarCall::reloc_destination(address orig_address) {
+address NativeFarCall::reloc_destination() {
   address call_addr = instruction_address();
+  assert(NativeFarCall::is_at(call_addr), "unexpected code at call site");
 
   CodeBlob *code = CodeCache::find_blob(call_addr);
   assert(code != nullptr, "Could not find the containing code blob");
 
   address stub_addr = nullptr;
-  if (code != nullptr && code->is_nmethod()) {
-    stub_addr = trampoline_stub_Relocation::get_trampoline_for(call_addr, (nmethod*)code);
+  if (code->is_nmethod()) {
+    stub_addr = trampoline_stub_Relocation::get_trampoline_for(call_addr, code->as_nmethod());
   }
 
   if (stub_addr != nullptr) {
     stub_addr = MacroAssembler::target_addr_for_insn(call_addr);
   }
+
   return stub_addr;
 }
 
@@ -128,18 +130,13 @@ void NativeFarCall::print() {
   tty->print_cr(PTR_FORMAT ": auipc,ld,jalr x1, offset/reg, ", p2i(addr_at(0)));
 }
 
-bool NativeFarCall::set_destination_mt_safe(address dest, bool assert_lock) {
+bool NativeFarCall::set_destination_mt_safe(address dest) {
   assert(NativeFarCall::is_at(addr_at(0)), "unexpected code at call site");
-  assert(!assert_lock ||
-         (CodeCache_lock->is_locked() || SafepointSynchronize::is_at_safepoint()) ||
+  assert((CodeCache_lock->is_locked() || SafepointSynchronize::is_at_safepoint()) ||
          CompiledICLocker::is_safe(addr_at(0)),
          "concurrent code patching");
 
-  address call_addr = addr_at(0);
-  assert(NativeFarCall::is_at(call_addr), "unexpected code at call site");
-
   address stub_addr = stub_address();
-
   if (stub_addr != nullptr) {
     set_stub_address_destination_at(stub_addr, dest);
     return true;
@@ -156,10 +153,9 @@ bool NativeFarCall::reloc_set_destination(address dest) {
   assert(code != nullptr, "Could not find the containing code blob");
 
   address stub_addr = nullptr;
-  if (code != nullptr && code->is_nmethod()) {
-    stub_addr = trampoline_stub_Relocation::get_trampoline_for(call_addr, (nmethod*)code);
+  if (code->is_nmethod()) {
+    stub_addr = trampoline_stub_Relocation::get_trampoline_for(call_addr, code->as_nmethod());
   }
-
   if (stub_addr != nullptr) {
     MacroAssembler::pd_patch_instruction_size(call_addr, stub_addr);
   }
@@ -209,7 +205,7 @@ bool NativeFarCall::is_at(address addr) {
       (MacroAssembler::extract_rd(addr + instr_size)       == x6) &&
       (MacroAssembler::extract_rs1(addr + instr_size)      == x6) &&
       (MacroAssembler::extract_rs1(addr + 2 * instr_size)  == x6) &&
-      (MacroAssembler::extract_rd(addr + 2 * instr_size)  == x1)) {
+      (MacroAssembler::extract_rd(addr + 2 * instr_size)   == x1)) {
     return true;
   }
   return false;
@@ -238,8 +234,8 @@ address NativeCall::destination() const {
   return NativeFarCall::at(addr_at(0))->destination();
 }
 
-address NativeCall::reloc_destination(address orig_address) {
-  return NativeFarCall::at(addr_at(0))->reloc_destination(orig_address);
+address NativeCall::reloc_destination() {
+  return NativeFarCall::at(addr_at(0))->reloc_destination();
 }
 
 void NativeCall::set_destination(address dest) {
@@ -254,8 +250,8 @@ void NativeCall::print() {
   NativeFarCall::at(addr_at(0))->print();;
 }
 
-bool NativeCall::set_destination_mt_safe(address dest, bool assert_lock) {
-  return NativeFarCall::at(addr_at(0))->set_destination_mt_safe(dest, assert_lock);
+bool NativeCall::set_destination_mt_safe(address dest) {
+  return NativeFarCall::at(addr_at(0))->set_destination_mt_safe(dest);
 }
 
 bool NativeCall::reloc_set_destination(address dest) {

src/hotspot/cpu/riscv/nativeInst_riscv.hpp

@@ -134,14 +134,14 @@ class NativeCall: private NativeInstruction {
   address next_instruction_address() const;
   address return_address() const;
   address destination() const;
-  address reloc_destination(address orig_address);
+  address reloc_destination();
 
   void verify_alignment() {} // do nothing on riscv
   void verify();
   void print();
 
   void set_destination(address dest);
-  bool set_destination_mt_safe(address dest, bool assert_lock = true);
+  bool set_destination_mt_safe(address dest);
   bool reloc_set_destination(address dest);
 
   static bool is_at(address addr);

src/hotspot/cpu/riscv/relocInfo_riscv.cpp

@@ -72,13 +72,12 @@ void Relocation::pd_set_data_value(address x, bool verify_only) {
 }
 
 address Relocation::pd_call_destination(address orig_addr) {
-  assert(is_call(), "should be an address instruction here");
+  assert(is_call(), "should be a call here");
   if (NativeCall::is_at(addr())) {
-    return nativeCall_at(addr())->reloc_destination(orig_addr);
+    return nativeCall_at(addr())->reloc_destination();
   }
-  // Non call reloc
+
   if (orig_addr != nullptr) {
-    // the extracted address from the instructions in address orig_addr
     address new_addr = MacroAssembler::pd_call_destination(orig_addr);
     // If call is branch to self, don't try to relocate it, just leave it
     // as branch to self. This happens during code generation if the code
@@ -87,20 +86,19 @@ address Relocation::pd_call_destination(address orig_addr) {
     new_addr = (new_addr == orig_addr) ? addr() : new_addr;
     return new_addr;
   }
+
   return MacroAssembler::pd_call_destination(addr());
 }
 
 void Relocation::pd_set_call_destination(address x) {
-  assert(is_call(), "should be an address instruction here");
+  assert(is_call(), "should be a call here");
   if (NativeCall::is_at(addr())) {
-    NativeCall* nc = nativeCall_at(addr());
-    if (nc->reloc_set_destination(x)) {
-      return;
-    }
+    NativeCall* call = nativeCall_at(addr());
+    call->reloc_set_destination(x);
+  } else {
+    MacroAssembler::pd_patch_instruction_size(addr(), x);
+    assert(pd_call_destination(addr()) == x, "fail in reloc");
   }
-  MacroAssembler::pd_patch_instruction_size(addr(), x);
-  address pd_call = pd_call_destination(addr());
-  assert(pd_call == x, "fail in reloc");
 }
 
 address* Relocation::pd_address_in_code() {

src/hotspot/cpu/zero/stubGenerator_zero.cpp

@@ -180,6 +180,10 @@ class StubGenerator: public StubCodeGenerator {
 
   void generate_preuniverse_stubs() {
     StubRoutines::_fence_entry               = ShouldNotCallThisStub();
+    StubRoutines::_atomic_xchg_entry         = ShouldNotCallThisStub();
+    StubRoutines::_atomic_cmpxchg_entry      = ShouldNotCallThisStub();
+    StubRoutines::_atomic_cmpxchg_long_entry = ShouldNotCallThisStub();
+    StubRoutines::_atomic_add_entry          = ShouldNotCallThisStub();
   }
 
   void generate_initial_stubs() {
@@ -192,12 +196,6 @@ class StubGenerator: public StubCodeGenerator {
     StubRoutines::_forward_exception_entry   = ShouldNotCallThisStub();
     StubRoutines::_call_stub_entry           = (address) call_stub;
     StubRoutines::_catch_exception_entry     = ShouldNotCallThisStub();
-
-    // atomic calls
-    StubRoutines::_atomic_xchg_entry         = ShouldNotCallThisStub();
-    StubRoutines::_atomic_cmpxchg_entry      = ShouldNotCallThisStub();
-    StubRoutines::_atomic_cmpxchg_long_entry = ShouldNotCallThisStub();
-    StubRoutines::_atomic_add_entry          = ShouldNotCallThisStub();
   }
 
   void generate_continuation_stubs() {

src/hotspot/share/gc/shared/parallelCleaning.cpp

@@ -128,6 +128,6 @@ void KlassCleaningTask::work() {
   // All workers will help cleaning the classes,
   InstanceKlass* klass;
   while ((klass = claim_next_klass()) != nullptr) {
-    clean_klass(klass);
+    Klass::clean_weak_instanceklass_links(klass);
   }
 }

src/hotspot/share/gc/shared/parallelCleaning.hpp

@@ -66,10 +66,6 @@ class KlassCleaningTask : public StackObj {
 
 public:
 
-  void clean_klass(InstanceKlass* ik) {
-    ik->clean_weak_instanceklass_links();
-  }
-
   void work();
 };
 

src/hotspot/share/oops/klass.cpp

@@ -749,17 +749,20 @@ void Klass::clean_weak_klass_links(bool unloading_occurred, bool clean_alive_kla
     // Clean the implementors list and method data.
     if (clean_alive_klasses && current->is_instance_klass()) {
       InstanceKlass* ik = InstanceKlass::cast(current);
-      ik->clean_weak_instanceklass_links();
-
-      // JVMTI RedefineClasses creates previous versions that are not in
-      // the class hierarchy, so process them here.
-      while ((ik = ik->previous_versions()) != nullptr) {
-        ik->clean_weak_instanceklass_links();
-      }
+      clean_weak_instanceklass_links(ik);
     }
   }
 }
 
+void Klass::clean_weak_instanceklass_links(InstanceKlass* ik) {
+  ik->clean_weak_instanceklass_links();
+  // JVMTI RedefineClasses creates previous versions that are not in
+  // the class hierarchy, so process them here.
+  while ((ik = ik->previous_versions()) != nullptr) {
+    ik->clean_weak_instanceklass_links();
+  }
+}
+
 void Klass::metaspace_pointers_do(MetaspaceClosure* it) {
   if (log_is_enabled(Trace, aot)) {
     ResourceMark rm;

src/hotspot/share/oops/klass.hpp

@@ -734,7 +734,10 @@ class Klass : public Metadata {
 
   void clean_subklass();
 
+  // Clean out unnecessary weak klass links from the whole klass hierarchy.
   static void clean_weak_klass_links(bool unloading_occurred, bool clean_alive_klasses = true);
+  // Clean out unnecessary weak klass links from the given InstanceKlass.
+  static void clean_weak_instanceklass_links(InstanceKlass* ik);
 
   // Return self, except for abstract classes with exactly 1
   // implementor.  Then return the 1 concrete implementation.