From 07fce8eff207eedcbab29b52660f19333df7c574 Mon Sep 17 00:00:00 2001 From: Albert Mingkun Yang Date: Tue, 9 Jan 2024 06:11:44 +0000 Subject: [PATCH] 8320864: Serial: Extract out Full GC related fields from ContiguousSpace Reviewed-by: kbarrett, sjohanss --- src/hotspot/share/gc/serial/genMarkSweep.cpp | 452 ++++++++++++++---- src/hotspot/share/gc/serial/genMarkSweep.hpp | 9 +- src/hotspot/share/gc/serial/generation.cpp | 31 -- src/hotspot/share/gc/serial/generation.hpp | 9 +- .../share/gc/serial/tenuredGeneration.hpp | 4 +- .../share/gc/shared/genCollectedHeap.cpp | 13 - .../share/gc/shared/genCollectedHeap.hpp | 12 - src/hotspot/share/gc/shared/space.cpp | 239 +-------- src/hotspot/share/gc/shared/space.hpp | 87 +--- src/hotspot/share/gc/shared/space.inline.hpp | 91 +--- src/hotspot/share/gc/shared/vmStructs_gc.hpp | 4 - 11 files changed, 363 insertions(+), 588 deletions(-) diff --git a/src/hotspot/share/gc/serial/genMarkSweep.cpp b/src/hotspot/share/gc/serial/genMarkSweep.cpp index fb55116ab816b..28ba4615f73a2 100644 --- a/src/hotspot/share/gc/serial/genMarkSweep.cpp +++ b/src/hotspot/share/gc/serial/genMarkSweep.cpp @@ -36,6 +36,7 @@ #include "gc/serial/defNewGeneration.hpp" #include "gc/serial/generation.hpp" #include "gc/serial/genMarkSweep.hpp" +#include "gc/serial/markSweep.inline.hpp" #include "gc/serial/serialGcRefProcProxyTask.hpp" #include "gc/serial/serialHeap.hpp" #include "gc/shared/classUnloadingContext.hpp" @@ -48,7 +49,7 @@ #include "gc/shared/preservedMarks.inline.hpp" #include "gc/shared/referencePolicy.hpp" #include "gc/shared/referenceProcessorPhaseTimes.hpp" -#include "gc/shared/space.hpp" +#include "gc/shared/space.inline.hpp" #include "gc/shared/strongRootsScope.hpp" #include "gc/shared/weakProcessor.hpp" #include "memory/universe.hpp" @@ -57,6 +58,7 @@ #include "prims/jvmtiExport.hpp" #include "runtime/handles.inline.hpp" #include "runtime/javaThread.hpp" +#include "runtime/prefetch.inline.hpp" #include "runtime/synchronizer.hpp" #include "runtime/vmThread.hpp" #include "utilities/copy.hpp" @@ -66,98 +68,281 @@ #include "jvmci/jvmci.hpp" #endif -void GenMarkSweep::invoke_at_safepoint(bool clear_all_softrefs) { - assert(SafepointSynchronize::is_at_safepoint(), "must be at a safepoint"); +class DeadSpacer : StackObj { + size_t _allowed_deadspace_words; + bool _active; + ContiguousSpace* _space; - SerialHeap* gch = SerialHeap::heap(); -#ifdef ASSERT - if (gch->soft_ref_policy()->should_clear_all_soft_refs()) { - assert(clear_all_softrefs, "Policy should have been checked earlier"); +public: + DeadSpacer(ContiguousSpace* space) : _allowed_deadspace_words(0), _space(space) { + size_t ratio = _space->allowed_dead_ratio(); + _active = ratio > 0; + + if (_active) { + // We allow some amount of garbage towards the bottom of the space, so + // we don't start compacting before there is a significant gain to be made. + // Occasionally, we want to ensure a full compaction, which is determined + // by the MarkSweepAlwaysCompactCount parameter. + if ((MarkSweep::total_invocations() % MarkSweepAlwaysCompactCount) != 0) { + _allowed_deadspace_words = (space->capacity() * ratio / 100) / HeapWordSize; + } else { + _active = false; + } + } } -#endif - gch->trace_heap_before_gc(_gc_tracer); - - // Increment the invocation count - _total_invocations++; + bool insert_deadspace(HeapWord* dead_start, HeapWord* dead_end) { + if (!_active) { + return false; + } - // Capture used regions for each generation that will be - // subject to collection, so that card table adjustments can - // be made intelligently (see clear / invalidate further below). - gch->save_used_regions(); + size_t dead_length = pointer_delta(dead_end, dead_start); + if (_allowed_deadspace_words >= dead_length) { + _allowed_deadspace_words -= dead_length; + CollectedHeap::fill_with_object(dead_start, dead_length); + oop obj = cast_to_oop(dead_start); + // obj->set_mark(obj->mark().set_marked()); + + assert(dead_length == obj->size(), "bad filler object size"); + log_develop_trace(gc, compaction)("Inserting object to dead space: " PTR_FORMAT ", " PTR_FORMAT ", " SIZE_FORMAT "b", + p2i(dead_start), p2i(dead_end), dead_length * HeapWordSize); + + return true; + } else { + _active = false; + return false; + } + } +}; - allocate_stacks(); +// Implement the "compaction" part of the mark-compact GC algorithm. +class Compacter { + // There are four spaces in total, but only the first three can be used after + // compact. IOW, old and eden/from must be enough for all live objs + static constexpr uint max_num_spaces = 4; + + struct CompactionSpace { + ContiguousSpace* _space; + // Will be the new top after compaction is complete. + HeapWord* _compaction_top; + // The first dead word in this contiguous space. It's an optimization to + // skip large chunk of live objects at the beginning. + HeapWord* _first_dead; + + void init(ContiguousSpace* space) { + _space = space; + _compaction_top = space->bottom(); + _first_dead = nullptr; + } + }; - mark_sweep_phase1(clear_all_softrefs); + CompactionSpace _spaces[max_num_spaces]; + // The num of spaces to be compacted, i.e. containing live objs. + uint _num_spaces; - mark_sweep_phase2(); + uint _index; - // Don't add any more derived pointers during phase3 -#if COMPILER2_OR_JVMCI - assert(DerivedPointerTable::is_active(), "Sanity"); - DerivedPointerTable::set_active(false); -#endif + HeapWord* get_compaction_top(uint index) const { + return _spaces[index]._compaction_top; + } - mark_sweep_phase3(); + HeapWord* get_first_dead(uint index) const { + return _spaces[index]._first_dead; + } - mark_sweep_phase4(); + ContiguousSpace* get_space(uint index) const { + return _spaces[index]._space; + } - restore_marks(); + void record_first_dead(uint index, HeapWord* first_dead) { + assert(_spaces[index]._first_dead == nullptr, "should write only once"); + _spaces[index]._first_dead = first_dead; + } - // Set saved marks for allocation profiler (and other things? -- dld) - // (Should this be in general part?) - gch->save_marks(); + HeapWord* alloc(size_t words) { + while (true) { + if (words <= pointer_delta(_spaces[_index]._space->end(), + _spaces[_index]._compaction_top)) { + HeapWord* result = _spaces[_index]._compaction_top; + _spaces[_index]._compaction_top += words; + if (_index == 0) { + // old-gen requires BOT update + static_cast(_spaces[0]._space)->update_for_block(result, result + words); + } + return result; + } + + // out-of-memory in this space + _index++; + assert(_index < max_num_spaces - 1, "the last space should not be used"); + } + } - deallocate_stacks(); + static void prefetch_read_scan(void* p) { + if (PrefetchScanIntervalInBytes >= 0) { + Prefetch::read(p, PrefetchScanIntervalInBytes); + } + } - MarkSweep::_string_dedup_requests->flush(); + static void prefetch_write_scan(void* p) { + if (PrefetchScanIntervalInBytes >= 0) { + Prefetch::write(p, PrefetchScanIntervalInBytes); + } + } - bool is_young_gen_empty = (gch->young_gen()->used() == 0); - gch->rem_set()->maintain_old_to_young_invariant(gch->old_gen(), is_young_gen_empty); + static void prefetch_write_copy(void* p) { + if (PrefetchCopyIntervalInBytes >= 0) { + Prefetch::write(p, PrefetchCopyIntervalInBytes); + } + } - gch->prune_scavengable_nmethods(); + static void forward_obj(oop obj, HeapWord* new_addr) { + prefetch_write_scan(obj); + if (cast_from_oop(obj) != new_addr) { + obj->forward_to(cast_to_oop(new_addr)); + } else { + assert(obj->is_gc_marked(), "inv"); + // This obj will stay in-place. Fix the markword. + obj->init_mark(); + } + } - // Update heap occupancy information which is used as - // input to soft ref clearing policy at the next gc. - Universe::heap()->update_capacity_and_used_at_gc(); + static HeapWord* find_next_live_addr(HeapWord* start, HeapWord* end) { + for (HeapWord* i_addr = start; i_addr < end; /* empty */) { + prefetch_read_scan(i_addr); + oop obj = cast_to_oop(i_addr); + if (obj->is_gc_marked()) { + return i_addr; + } + i_addr += obj->size(); + } + return end; + }; - // Signal that we have completed a visit to all live objects. - Universe::heap()->record_whole_heap_examined_timestamp(); + static size_t relocate(HeapWord* addr) { + // Prefetch source and destination + prefetch_read_scan(addr); - gch->trace_heap_after_gc(_gc_tracer); -} + oop obj = cast_to_oop(addr); + oop new_obj = obj->forwardee(); + HeapWord* new_addr = cast_from_oop(new_obj); + assert(addr != new_addr, "inv"); + prefetch_write_copy(new_addr); -void GenMarkSweep::allocate_stacks() { - void* scratch = nullptr; - size_t num_words; - DefNewGeneration* young_gen = (DefNewGeneration*)SerialHeap::heap()->young_gen(); - young_gen->contribute_scratch(scratch, num_words); + size_t obj_size = obj->size(); + Copy::aligned_conjoint_words(addr, new_addr, obj_size); + new_obj->init_mark(); - if (scratch != nullptr) { - _preserved_count_max = num_words * HeapWordSize / sizeof(PreservedMark); - } else { - _preserved_count_max = 0; + return obj_size; } - _preserved_marks = (PreservedMark*)scratch; - _preserved_count = 0; - - _preserved_overflow_stack_set.init(1); -} +public: + explicit Compacter(SerialHeap* heap) { + // In this order so that heap is compacted towards old-gen. + _spaces[0].init(heap->old_gen()->space()); + _spaces[1].init(heap->young_gen()->eden()); + _spaces[2].init(heap->young_gen()->from()); + + bool is_promotion_failed = (heap->young_gen()->from()->next_compaction_space() != nullptr); + if (is_promotion_failed) { + _spaces[3].init(heap->young_gen()->to()); + _num_spaces = 4; + } else { + _num_spaces = 3; + } + _index = 0; + } + void phase2_calculate_new_addr() { + for (uint i = 0; i < _num_spaces; ++i) { + ContiguousSpace* space = get_space(i); + HeapWord* cur_addr = space->bottom(); + HeapWord* top = space->top(); + + bool record_first_dead_done = false; + + DeadSpacer dead_spacer(space); + + while (cur_addr < top) { + oop obj = cast_to_oop(cur_addr); + size_t obj_size = obj->size(); + if (obj->is_gc_marked()) { + HeapWord* new_addr = alloc(obj_size); + forward_obj(obj, new_addr); + cur_addr += obj_size; + } else { + // Skipping the current known-unmarked obj + HeapWord* next_live_addr = find_next_live_addr(cur_addr + obj_size, top); + if (dead_spacer.insert_deadspace(cur_addr, next_live_addr)) { + // Register space for the filler obj + alloc(pointer_delta(next_live_addr, cur_addr)); + } else { + if (!record_first_dead_done) { + record_first_dead(i, cur_addr); + record_first_dead_done = true; + } + *(HeapWord**)cur_addr = next_live_addr; + } + cur_addr = next_live_addr; + } + } + + if (!record_first_dead_done) { + record_first_dead(i, top); + } + } + } -void GenMarkSweep::deallocate_stacks() { - if (_preserved_count_max != 0) { - DefNewGeneration* young_gen = (DefNewGeneration*)SerialHeap::heap()->young_gen(); - young_gen->reset_scratch(); + void phase3_adjust_pointers() { + for (uint i = 0; i < _num_spaces; ++i) { + ContiguousSpace* space = get_space(i); + HeapWord* cur_addr = space->bottom(); + HeapWord* const top = space->top(); + HeapWord* const first_dead = get_first_dead(i); + + while (cur_addr < top) { + prefetch_write_scan(cur_addr); + if (cur_addr < first_dead || cast_to_oop(cur_addr)->is_gc_marked()) { + size_t size = MarkSweep::adjust_pointers(cast_to_oop(cur_addr)); + cur_addr += size; + } else { + assert(*(HeapWord**)cur_addr > cur_addr, "forward progress"); + cur_addr = *(HeapWord**)cur_addr; + } + } + } } - _preserved_overflow_stack_set.reclaim(); - _marking_stack.clear(); - _objarray_stack.clear(true); -} + void phase4_compact() { + for (uint i = 0; i < _num_spaces; ++i) { + ContiguousSpace* space = get_space(i); + HeapWord* cur_addr = space->bottom(); + HeapWord* top = space->top(); + + // Check if the first obj inside this space is forwarded. + if (!cast_to_oop(cur_addr)->is_forwarded()) { + // Jump over consecutive (in-place) live-objs-chunk + cur_addr = get_first_dead(i); + } + + while (cur_addr < top) { + if (!cast_to_oop(cur_addr)->is_forwarded()) { + cur_addr = *(HeapWord**) cur_addr; + continue; + } + cur_addr += relocate(cur_addr); + } + + // Reset top and unused memory + space->set_top(get_compaction_top(i)); + if (ZapUnusedHeapArea) { + space->mangle_unused_area(); + } + } + } +}; -void GenMarkSweep::mark_sweep_phase1(bool clear_all_softrefs) { +void GenMarkSweep::phase1_mark(bool clear_all_softrefs) { // Recursively traverse all live objects and mark them GCTraceTime(Info, gc, phases) tm("Phase 1: Mark live objects", _gc_timer); @@ -241,54 +426,121 @@ void GenMarkSweep::mark_sweep_phase1(bool clear_all_softrefs) { } } +void GenMarkSweep::invoke_at_safepoint(bool clear_all_softrefs) { + assert(SafepointSynchronize::is_at_safepoint(), "must be at a safepoint"); -void GenMarkSweep::mark_sweep_phase2() { - // Now all live objects are marked, compute the new object addresses. - GCTraceTime(Info, gc, phases) tm("Phase 2: Compute new object addresses", _gc_timer); + SerialHeap* gch = SerialHeap::heap(); +#ifdef ASSERT + if (gch->soft_ref_policy()->should_clear_all_soft_refs()) { + assert(clear_all_softrefs, "Policy should have been checked earlier"); + } +#endif - SerialHeap::heap()->prepare_for_compaction(); -} + gch->trace_heap_before_gc(_gc_tracer); -class GenAdjustPointersClosure: public SerialHeap::GenClosure { -public: - void do_generation(Generation* gen) { - gen->adjust_pointers(); + // Increment the invocation count + _total_invocations++; + + // Capture used regions for each generation that will be + // subject to collection, so that card table adjustments can + // be made intelligently (see clear / invalidate further below). + gch->save_used_regions(); + + allocate_stacks(); + + phase1_mark(clear_all_softrefs); + + Compacter compacter{gch}; + + { + // Now all live objects are marked, compute the new object addresses. + GCTraceTime(Info, gc, phases) tm("Phase 2: Compute new object addresses", _gc_timer); + + compacter.phase2_calculate_new_addr(); } -}; -void GenMarkSweep::mark_sweep_phase3() { - SerialHeap* gch = SerialHeap::heap(); + // Don't add any more derived pointers during phase3 +#if COMPILER2_OR_JVMCI + assert(DerivedPointerTable::is_active(), "Sanity"); + DerivedPointerTable::set_active(false); +#endif + + { + // Adjust the pointers to reflect the new locations + GCTraceTime(Info, gc, phases) tm("Phase 3: Adjust pointers", gc_timer()); + + ClassLoaderDataGraph::verify_claimed_marks_cleared(ClassLoaderData::_claim_stw_fullgc_adjust); + + CodeBlobToOopClosure code_closure(&adjust_pointer_closure, CodeBlobToOopClosure::FixRelocations); + gch->process_roots(SerialHeap::SO_AllCodeCache, + &adjust_pointer_closure, + &adjust_cld_closure, + &adjust_cld_closure, + &code_closure); - // Adjust the pointers to reflect the new locations - GCTraceTime(Info, gc, phases) tm("Phase 3: Adjust pointers", gc_timer()); + WeakProcessor::oops_do(&adjust_pointer_closure); - ClassLoaderDataGraph::verify_claimed_marks_cleared(ClassLoaderData::_claim_stw_fullgc_adjust); + adjust_marks(); + compacter.phase3_adjust_pointers(); + } - CodeBlobToOopClosure code_closure(&adjust_pointer_closure, CodeBlobToOopClosure::FixRelocations); - gch->process_roots(SerialHeap::SO_AllCodeCache, - &adjust_pointer_closure, - &adjust_cld_closure, - &adjust_cld_closure, - &code_closure); + { + // All pointers are now adjusted, move objects accordingly + GCTraceTime(Info, gc, phases) tm("Phase 4: Move objects", _gc_timer); - gch->gen_process_weak_roots(&adjust_pointer_closure); + compacter.phase4_compact(); + } - adjust_marks(); - GenAdjustPointersClosure blk; - gch->generation_iterate(&blk, true); + restore_marks(); + + // Set saved marks for allocation profiler (and other things? -- dld) + // (Should this be in general part?) + gch->save_marks(); + + deallocate_stacks(); + + MarkSweep::_string_dedup_requests->flush(); + + bool is_young_gen_empty = (gch->young_gen()->used() == 0); + gch->rem_set()->maintain_old_to_young_invariant(gch->old_gen(), is_young_gen_empty); + + gch->prune_scavengable_nmethods(); + + // Update heap occupancy information which is used as + // input to soft ref clearing policy at the next gc. + Universe::heap()->update_capacity_and_used_at_gc(); + + // Signal that we have completed a visit to all live objects. + Universe::heap()->record_whole_heap_examined_timestamp(); + + gch->trace_heap_after_gc(_gc_tracer); } -class GenCompactClosure: public SerialHeap::GenClosure { -public: - void do_generation(Generation* gen) { - gen->compact(); +void GenMarkSweep::allocate_stacks() { + void* scratch = nullptr; + size_t num_words; + DefNewGeneration* young_gen = (DefNewGeneration*)SerialHeap::heap()->young_gen(); + young_gen->contribute_scratch(scratch, num_words); + + if (scratch != nullptr) { + _preserved_count_max = num_words * HeapWordSize / sizeof(PreservedMark); + } else { + _preserved_count_max = 0; } -}; -void GenMarkSweep::mark_sweep_phase4() { - // All pointers are now adjusted, move objects accordingly - GCTraceTime(Info, gc, phases) tm("Phase 4: Move objects", _gc_timer); + _preserved_marks = (PreservedMark*)scratch; + _preserved_count = 0; + + _preserved_overflow_stack_set.init(1); +} - GenCompactClosure blk; - SerialHeap::heap()->generation_iterate(&blk, true); +void GenMarkSweep::deallocate_stacks() { + if (_preserved_count_max != 0) { + DefNewGeneration* young_gen = (DefNewGeneration*)SerialHeap::heap()->young_gen(); + young_gen->reset_scratch(); + } + + _preserved_overflow_stack_set.reclaim(); + _marking_stack.clear(); + _objarray_stack.clear(true); } diff --git a/src/hotspot/share/gc/serial/genMarkSweep.hpp b/src/hotspot/share/gc/serial/genMarkSweep.hpp index be830318bd0d9..520e801057389 100644 --- a/src/hotspot/share/gc/serial/genMarkSweep.hpp +++ b/src/hotspot/share/gc/serial/genMarkSweep.hpp @@ -32,15 +32,8 @@ class GenMarkSweep : public MarkSweep { static void invoke_at_safepoint(bool clear_all_softrefs); private: - // Mark live objects - static void mark_sweep_phase1(bool clear_all_softrefs); - // Calculate new addresses - static void mark_sweep_phase2(); - // Update pointers - static void mark_sweep_phase3(); - // Move objects to new positions - static void mark_sweep_phase4(); + static void phase1_mark(bool clear_all_softrefs); // Temporary data structures for traversal and storing/restoring marks static void allocate_stacks(); diff --git a/src/hotspot/share/gc/serial/generation.cpp b/src/hotspot/share/gc/serial/generation.cpp index aa3c804ba3c20..30f1429fd40f2 100644 --- a/src/hotspot/share/gc/serial/generation.cpp +++ b/src/hotspot/share/gc/serial/generation.cpp @@ -218,34 +218,3 @@ void Generation::object_iterate(ObjectClosure* cl) { GenerationObjIterateClosure blk(cl); space_iterate(&blk); } - -void Generation::prepare_for_compaction(CompactPoint* cp) { - // Generic implementation, can be specialized - ContiguousSpace* space = first_compaction_space(); - while (space != nullptr) { - space->prepare_for_compaction(cp); - space = space->next_compaction_space(); - } -} - -class AdjustPointersClosure: public SpaceClosure { - public: - void do_space(Space* sp) { - sp->adjust_pointers(); - } -}; - -void Generation::adjust_pointers() { - // Note that this is done over all spaces, not just the compactible - // ones. - AdjustPointersClosure blk; - space_iterate(&blk, true); -} - -void Generation::compact() { - ContiguousSpace* sp = first_compaction_space(); - while (sp != nullptr) { - sp->compact(); - sp = sp->next_compaction_space(); - } -} diff --git a/src/hotspot/share/gc/serial/generation.hpp b/src/hotspot/share/gc/serial/generation.hpp index 63eefae720bbb..d93c2b66df717 100644 --- a/src/hotspot/share/gc/serial/generation.hpp +++ b/src/hotspot/share/gc/serial/generation.hpp @@ -51,7 +51,7 @@ class DefNewGeneration; class GCMemoryManager; class ContiguousSpace; -class CompactPoint; + class OopClosure; class GCStats; @@ -286,13 +286,6 @@ class Generation: public CHeapObj { GCStats* gc_stats() const { return _gc_stats; } virtual void update_gc_stats(Generation* current_generation, bool full) {} - // Mark sweep support phase2 - virtual void prepare_for_compaction(CompactPoint* cp); - // Mark sweep support phase3 - virtual void adjust_pointers(); - // Mark sweep support phase4 - virtual void compact(); - // Accessing "marks". // This function gives a generation a chance to note a point between diff --git a/src/hotspot/share/gc/serial/tenuredGeneration.hpp b/src/hotspot/share/gc/serial/tenuredGeneration.hpp index 75e8bf8486051..6b2a5891c64cd 100644 --- a/src/hotspot/share/gc/serial/tenuredGeneration.hpp +++ b/src/hotspot/share/gc/serial/tenuredGeneration.hpp @@ -70,8 +70,6 @@ class TenuredGeneration: public Generation { GenerationCounters* _gen_counters; CSpaceCounters* _space_counters; - // Accessing spaces - TenuredSpace* space() const { return _the_space; } // Attempt to expand the generation by "bytes". Expand by at a // minimum "expand_bytes". Return true if some amount (not @@ -85,6 +83,8 @@ class TenuredGeneration: public Generation { public: virtual void compute_new_size(); + TenuredSpace* space() const { return _the_space; } + // Grow generation with specified size (returns false if unable to grow) bool grow_by(size_t bytes); // Grow generation to reserved size. diff --git a/src/hotspot/share/gc/shared/genCollectedHeap.cpp b/src/hotspot/share/gc/shared/genCollectedHeap.cpp index f4c21cce6dcda..cf101caae38b3 100644 --- a/src/hotspot/share/gc/shared/genCollectedHeap.cpp +++ b/src/hotspot/share/gc/shared/genCollectedHeap.cpp @@ -713,10 +713,6 @@ void GenCollectedHeap::process_roots(ScanningOption so, DEBUG_ONLY(ScavengableNMethods::asserted_non_scavengable_nmethods_do(&assert_code_is_non_scavengable)); } -void GenCollectedHeap::gen_process_weak_roots(OopClosure* root_closure) { - WeakProcessor::oops_do(root_closure); -} - bool GenCollectedHeap::no_allocs_since_save_marks() { return _young_gen->no_allocs_since_save_marks() && _old_gen->no_allocs_since_save_marks(); @@ -911,15 +907,6 @@ GenCollectedHeap* GenCollectedHeap::heap() { return named_heap(CollectedHeap::Serial); } -#if INCLUDE_SERIALGC -void GenCollectedHeap::prepare_for_compaction() { - // Start by compacting into same gen. - CompactPoint cp(_old_gen); - _old_gen->prepare_for_compaction(&cp); - _young_gen->prepare_for_compaction(&cp); -} -#endif // INCLUDE_SERIALGC - void GenCollectedHeap::verify(VerifyOption option /* ignored */) { log_debug(gc, verify)("%s", _old_gen->name()); _old_gen->verify(); diff --git a/src/hotspot/share/gc/shared/genCollectedHeap.hpp b/src/hotspot/share/gc/shared/genCollectedHeap.hpp index 72548f6e2f54a..1c1759f1422be 100644 --- a/src/hotspot/share/gc/shared/genCollectedHeap.hpp +++ b/src/hotspot/share/gc/shared/genCollectedHeap.hpp @@ -292,11 +292,6 @@ class GenCollectedHeap : public CollectedHeap { CLDClosure* weak_cld_closure, CodeBlobToOopClosure* code_roots); - // Apply "root_closure" to all the weak roots of the system. - // These include JNI weak roots, string table, - // and referents of reachable weak refs. - void gen_process_weak_roots(OopClosure* root_closure); - // Set the saved marks of generations, if that makes sense. // In particular, if any generation might iterate over the oops // in other generations, it should call this method. @@ -340,13 +335,6 @@ class GenCollectedHeap : public CollectedHeap { HeapWord* mem_allocate_work(size_t size, bool is_tlab); -#if INCLUDE_SERIALGC - // For use by mark-sweep. As implemented, mark-sweep-compact is global - // in an essential way: compaction is performed across generations, by - // iterating over spaces. - void prepare_for_compaction(); -#endif - // Save the tops of the spaces in all generations void record_gen_tops_before_GC() PRODUCT_RETURN; diff --git a/src/hotspot/share/gc/shared/space.cpp b/src/hotspot/share/gc/shared/space.cpp index 1c2afaa8b70ea..b6dc4e1e7d8f2 100644 --- a/src/hotspot/share/gc/shared/space.cpp +++ b/src/hotspot/share/gc/shared/space.cpp @@ -35,19 +35,13 @@ #include "oops/oop.inline.hpp" #include "runtime/atomic.hpp" #include "runtime/java.hpp" -#include "runtime/prefetch.inline.hpp" #include "runtime/safepoint.hpp" #include "utilities/align.hpp" #include "utilities/copy.hpp" #include "utilities/globalDefinitions.hpp" #include "utilities/macros.hpp" -#if INCLUDE_SERIALGC -#include "gc/serial/serialBlockOffsetTable.inline.hpp" -#include "gc/serial/defNewGeneration.hpp" -#endif ContiguousSpace::ContiguousSpace(): Space(), - _compaction_top(nullptr), _next_compaction_space(nullptr), _top(nullptr) { _mangler = new GenSpaceMangler(this); @@ -59,8 +53,7 @@ ContiguousSpace::~ContiguousSpace() { void ContiguousSpace::initialize(MemRegion mr, bool clear_space, - bool mangle_space) -{ + bool mangle_space) { HeapWord* bottom = mr.start(); HeapWord* end = mr.end(); assert(Universe::on_page_boundary(bottom) && Universe::on_page_boundary(end), @@ -70,7 +63,6 @@ void ContiguousSpace::initialize(MemRegion mr, if (clear_space) { clear(mangle_space); } - set_compaction_top(bottom); _next_compaction_space = nullptr; } @@ -80,7 +72,6 @@ void ContiguousSpace::clear(bool mangle_space) { if (ZapUnusedHeapArea && mangle_space) { mangle_unused_area(); } - _compaction_top = bottom(); } bool ContiguousSpace::is_free_block(const HeapWord* p) const { @@ -115,230 +106,6 @@ void ContiguousSpace::mangle_unused_area_complete() { #endif // NOT_PRODUCT -HeapWord* ContiguousSpace::forward(oop q, size_t size, - CompactPoint* cp, HeapWord* compact_top) { - // q is alive - // First check if we should switch compaction space - assert(this == cp->space, "'this' should be current compaction space."); - size_t compaction_max_size = pointer_delta(end(), compact_top); - while (size > compaction_max_size) { - // switch to next compaction space - cp->space->set_compaction_top(compact_top); - cp->space = cp->space->next_compaction_space(); - if (cp->space == nullptr) { - cp->gen = GenCollectedHeap::heap()->young_gen(); - assert(cp->gen != nullptr, "compaction must succeed"); - cp->space = cp->gen->first_compaction_space(); - assert(cp->space != nullptr, "generation must have a first compaction space"); - } - compact_top = cp->space->bottom(); - cp->space->set_compaction_top(compact_top); - compaction_max_size = pointer_delta(cp->space->end(), compact_top); - } - - // store the forwarding pointer into the mark word - if (cast_from_oop(q) != compact_top) { - q->forward_to(cast_to_oop(compact_top)); - assert(q->is_gc_marked(), "encoding the pointer should preserve the mark"); - } else { - // if the object isn't moving we can just set the mark to the default - // mark and handle it specially later on. - q->init_mark(); - assert(!q->is_forwarded(), "should not be forwarded"); - } - - compact_top += size; - - // We need to update the offset table so that the beginnings of objects can be - // found during scavenge. Note that we are updating the offset table based on - // where the object will be once the compaction phase finishes. - cp->space->update_for_block(compact_top - size, compact_top); - return compact_top; -} - -#if INCLUDE_SERIALGC - -void ContiguousSpace::prepare_for_compaction(CompactPoint* cp) { - // Compute the new addresses for the live objects and store it in the mark - // Used by universe::mark_sweep_phase2() - - // We're sure to be here before any objects are compacted into this - // space, so this is a good time to initialize this: - set_compaction_top(bottom()); - - if (cp->space == nullptr) { - assert(cp->gen != nullptr, "need a generation"); - assert(cp->gen->first_compaction_space() == this, "just checking"); - cp->space = cp->gen->first_compaction_space(); - cp->space->set_compaction_top(cp->space->bottom()); - } - - HeapWord* compact_top = cp->space->compaction_top(); // This is where we are currently compacting to. - - DeadSpacer dead_spacer(this); - - HeapWord* end_of_live = bottom(); // One byte beyond the last byte of the last live object. - HeapWord* first_dead = nullptr; // The first dead object. - - const intx interval = PrefetchScanIntervalInBytes; - - HeapWord* cur_obj = bottom(); - HeapWord* scan_limit = top(); - - while (cur_obj < scan_limit) { - if (cast_to_oop(cur_obj)->is_gc_marked()) { - // prefetch beyond cur_obj - Prefetch::write(cur_obj, interval); - size_t size = cast_to_oop(cur_obj)->size(); - compact_top = cp->space->forward(cast_to_oop(cur_obj), size, cp, compact_top); - cur_obj += size; - end_of_live = cur_obj; - } else { - // run over all the contiguous dead objects - HeapWord* end = cur_obj; - do { - // prefetch beyond end - Prefetch::write(end, interval); - end += cast_to_oop(end)->size(); - } while (end < scan_limit && !cast_to_oop(end)->is_gc_marked()); - - // see if we might want to pretend this object is alive so that - // we don't have to compact quite as often. - if (cur_obj == compact_top && dead_spacer.insert_deadspace(cur_obj, end)) { - oop obj = cast_to_oop(cur_obj); - compact_top = cp->space->forward(obj, obj->size(), cp, compact_top); - end_of_live = end; - } else { - // otherwise, it really is a free region. - - // cur_obj is a pointer to a dead object. Use this dead memory to store a pointer to the next live object. - *(HeapWord**)cur_obj = end; - - // see if this is the first dead region. - if (first_dead == nullptr) { - first_dead = cur_obj; - } - } - - // move on to the next object - cur_obj = end; - } - } - - assert(cur_obj == scan_limit, "just checking"); - _end_of_live = end_of_live; - if (first_dead != nullptr) { - _first_dead = first_dead; - } else { - _first_dead = end_of_live; - } - - // save the compaction_top of the compaction space. - cp->space->set_compaction_top(compact_top); -} - -void ContiguousSpace::adjust_pointers() { - // Check first is there is any work to do. - if (used() == 0) { - return; // Nothing to do. - } - - // adjust all the interior pointers to point at the new locations of objects - // Used by MarkSweep::mark_sweep_phase3() - - HeapWord* cur_obj = bottom(); - HeapWord* const end_of_live = _end_of_live; // Established by prepare_for_compaction(). - HeapWord* const first_dead = _first_dead; // Established by prepare_for_compaction(). - - assert(first_dead <= end_of_live, "Stands to reason, no?"); - - const intx interval = PrefetchScanIntervalInBytes; - - debug_only(HeapWord* prev_obj = nullptr); - while (cur_obj < end_of_live) { - Prefetch::write(cur_obj, interval); - if (cur_obj < first_dead || cast_to_oop(cur_obj)->is_gc_marked()) { - // cur_obj is alive - // point all the oops to the new location - size_t size = MarkSweep::adjust_pointers(cast_to_oop(cur_obj)); - debug_only(prev_obj = cur_obj); - cur_obj += size; - } else { - debug_only(prev_obj = cur_obj); - // cur_obj is not a live object, instead it points at the next live object - cur_obj = *(HeapWord**)cur_obj; - assert(cur_obj > prev_obj, "we should be moving forward through memory, cur_obj: " PTR_FORMAT ", prev_obj: " PTR_FORMAT, p2i(cur_obj), p2i(prev_obj)); - } - } - - assert(cur_obj == end_of_live, "just checking"); -} - -void ContiguousSpace::compact() { - // Copy all live objects to their new location - // Used by MarkSweep::mark_sweep_phase4() - - verify_up_to_first_dead(this); - - HeapWord* const start = bottom(); - HeapWord* const end_of_live = _end_of_live; - - assert(_first_dead <= end_of_live, "Invariant. _first_dead: " PTR_FORMAT " <= end_of_live: " PTR_FORMAT, p2i(_first_dead), p2i(end_of_live)); - if (_first_dead == end_of_live && (start == end_of_live || !cast_to_oop(start)->is_gc_marked())) { - // Nothing to compact. The space is either empty or all live object should be left in place. - clear_empty_region(this); - return; - } - - const intx scan_interval = PrefetchScanIntervalInBytes; - const intx copy_interval = PrefetchCopyIntervalInBytes; - - assert(start < end_of_live, "bottom: " PTR_FORMAT " should be < end_of_live: " PTR_FORMAT, p2i(start), p2i(end_of_live)); - HeapWord* cur_obj = start; - if (_first_dead > cur_obj && !cast_to_oop(cur_obj)->is_gc_marked()) { - // All object before _first_dead can be skipped. They should not be moved. - // A pointer to the first live object is stored at the memory location for _first_dead. - cur_obj = *(HeapWord**)(_first_dead); - } - - debug_only(HeapWord* prev_obj = nullptr); - while (cur_obj < end_of_live) { - if (!cast_to_oop(cur_obj)->is_forwarded()) { - debug_only(prev_obj = cur_obj); - // The first word of the dead object contains a pointer to the next live object or end of space. - cur_obj = *(HeapWord**)cur_obj; - assert(cur_obj > prev_obj, "we should be moving forward through memory"); - } else { - // prefetch beyond q - Prefetch::read(cur_obj, scan_interval); - - // size and destination - size_t size = cast_to_oop(cur_obj)->size(); - HeapWord* compaction_top = cast_from_oop(cast_to_oop(cur_obj)->forwardee()); - - // prefetch beyond compaction_top - Prefetch::write(compaction_top, copy_interval); - - // copy object and reinit its mark - assert(cur_obj != compaction_top, "everything in this pass should be moving"); - Copy::aligned_conjoint_words(cur_obj, compaction_top, size); - oop new_obj = cast_to_oop(compaction_top); - - ContinuationGCSupport::transform_stack_chunk(new_obj); - - new_obj->init_mark(); - assert(new_obj->klass() != nullptr, "should have a class"); - - debug_only(prev_obj = cur_obj); - cur_obj += size; - } - } - - clear_empty_region(this); -} - -#endif // INCLUDE_SERIALGC - void Space::print_short() const { print_short_on(tty); } void Space::print_short_on(outputStream* st) const { @@ -481,10 +248,6 @@ HeapWord* ContiguousSpace::par_allocate(size_t size) { } #if INCLUDE_SERIALGC -void TenuredSpace::update_for_block(HeapWord* start, HeapWord* end) { - _offsets.update_for_block(start, end); -} - HeapWord* TenuredSpace::block_start_const(const void* addr) const { HeapWord* cur_block = _offsets.block_start_reaching_into_card(addr); diff --git a/src/hotspot/share/gc/shared/space.hpp b/src/hotspot/share/gc/shared/space.hpp index 5d559f74f5eef..6b8279ec8bd05 100644 --- a/src/hotspot/share/gc/shared/space.hpp +++ b/src/hotspot/share/gc/shared/space.hpp @@ -184,29 +184,12 @@ class Space: public CHeapObj { // Allocation (return null if full). Enforces mutual exclusion internally. virtual HeapWord* par_allocate(size_t word_size) = 0; -#if INCLUDE_SERIALGC - // Mark-sweep-compact support: all spaces can update pointers to objects - // moving as a part of compaction. - virtual void adjust_pointers() = 0; -#endif - void print() const; virtual void print_on(outputStream* st) const; void print_short() const; void print_short_on(outputStream* st) const; }; -// A structure to represent a point at which objects are being copied -// during compaction. -class CompactPoint : public StackObj { -public: - Generation* gen; - ContiguousSpace* space; - - CompactPoint(Generation* g = nullptr) : - gen(g), space(nullptr) {} -}; - class GenSpaceMangler; // A space in which the free area is contiguous. It therefore supports @@ -215,26 +198,13 @@ class ContiguousSpace: public Space { friend class VMStructs; private: - HeapWord* _compaction_top; ContiguousSpace* _next_compaction_space; - static inline void verify_up_to_first_dead(ContiguousSpace* space) NOT_DEBUG_RETURN; - - static inline void clear_empty_region(ContiguousSpace* space); - - protected: +protected: HeapWord* _top; // A helper for mangling the unused area of the space in debug builds. GenSpaceMangler* _mangler; - // Used during compaction. - HeapWord* _first_dead; - HeapWord* _end_of_live; - - // This the function to invoke when an allocation of an object covering - // "start" to "end" occurs to update other internal data structures. - virtual void update_for_block(HeapWord* start, HeapWord* the_end) { } - GenSpaceMangler* mangler() { return _mangler; } // Allocation helpers (return null if full). @@ -254,23 +224,13 @@ class ContiguousSpace: public Space { // The "clear" method must be called on a region that may have // had allocation performed in it, but is now to be considered empty. - virtual void clear(bool mangle_space); - - // Used temporarily during a compaction phase to hold the value - // top should have when compaction is complete. - HeapWord* compaction_top() const { return _compaction_top; } - - void set_compaction_top(HeapWord* value) { - assert(value == nullptr || (value >= bottom() && value <= end()), - "should point inside space"); - _compaction_top = value; - } + void clear(bool mangle_space); // Returns the next space (in the current generation) to be compacted in // the global compaction order. Also is used to select the next // space into which to compact. - virtual ContiguousSpace* next_compaction_space() const { + ContiguousSpace* next_compaction_space() const { return _next_compaction_space; } @@ -278,42 +238,10 @@ class ContiguousSpace: public Space { _next_compaction_space = csp; } -#if INCLUDE_SERIALGC - // MarkSweep support phase2 - - // Start the process of compaction of the current space: compute - // post-compaction addresses, and insert forwarding pointers. The fields - // "cp->gen" and "cp->compaction_space" are the generation and space into - // which we are currently compacting. This call updates "cp" as necessary, - // and leaves the "compaction_top" of the final value of - // "cp->compaction_space" up-to-date. Offset tables may be updated in - // this phase as if the final copy had occurred; if so, "cp->threshold" - // indicates when the next such action should be taken. - void prepare_for_compaction(CompactPoint* cp); - // MarkSweep support phase3 - void adjust_pointers() override; - // MarkSweep support phase4 - virtual void compact(); -#endif // INCLUDE_SERIALGC - // The maximum percentage of objects that can be dead in the compacted // live part of a compacted space ("deadwood" support.) virtual size_t allowed_dead_ratio() const { return 0; }; - // "q" is an object of the given "size" that should be forwarded; - // "cp" names the generation ("gen") and containing "this" (which must - // also equal "cp->space"). "compact_top" is where in "this" the - // next object should be forwarded to. If there is room in "this" for - // the object, insert an appropriate forwarding pointer in "q". - // If not, go to the next compaction space (there must - // be one, since compaction must succeed -- we go to the first space of - // the previous generation if necessary, updating "cp"), reset compact_top - // and then forward. In either case, returns the new value of "compact_top". - // Invokes the "update_for_block" function of the then-current compaction - // space. - virtual HeapWord* forward(oop q, size_t size, CompactPoint* cp, - HeapWord* compact_top); - // Accessors HeapWord* top() const { return _top; } void set_top(HeapWord* value) { _top = value; } @@ -359,12 +287,6 @@ class ContiguousSpace: public Space { // Iteration void object_iterate(ObjectClosure* blk) override; - // Compaction support - void reset_after_compaction() { - assert(compaction_top() >= bottom() && compaction_top() <= end(), "should point inside space"); - set_top(compaction_top()); - } - // Apply "blk->do_oop" to the addresses of all reference fields in objects // starting with the _saved_mark_word, which was noted during a generation's // save_marks and is required to denote the head of an object. @@ -419,8 +341,7 @@ class TenuredSpace: public ContiguousSpace { inline HeapWord* allocate(size_t word_size) override; inline HeapWord* par_allocate(size_t word_size) override; - // MarkSweep support phase3 - void update_for_block(HeapWord* start, HeapWord* end) override; + inline void update_for_block(HeapWord* start, HeapWord* end); void print_on(outputStream* st) const override; }; diff --git a/src/hotspot/share/gc/shared/space.inline.hpp b/src/hotspot/share/gc/shared/space.inline.hpp index 9c4c2864cedb6..bfaf84e8fac01 100644 --- a/src/hotspot/share/gc/shared/space.inline.hpp +++ b/src/hotspot/share/gc/shared/space.inline.hpp @@ -27,17 +27,12 @@ #include "gc/shared/space.hpp" -#include "gc/serial/generation.hpp" #include "gc/shared/collectedHeap.hpp" #include "gc/shared/spaceDecorator.hpp" #include "oops/oop.inline.hpp" #include "oops/oopsHierarchy.hpp" #include "runtime/prefetch.inline.hpp" #include "runtime/safepoint.hpp" -#if INCLUDE_SERIALGC -#include "gc/serial/serialBlockOffsetTable.inline.hpp" -#include "gc/serial/markSweep.inline.hpp" -#endif inline HeapWord* Space::block_start(const void* p) { return block_start_const(p); @@ -60,90 +55,8 @@ inline HeapWord* TenuredSpace::par_allocate(size_t size) { return res; } -class DeadSpacer : StackObj { - size_t _allowed_deadspace_words; - bool _active; - ContiguousSpace* _space; - -public: - DeadSpacer(ContiguousSpace* space) : _allowed_deadspace_words(0), _space(space) { - size_t ratio = _space->allowed_dead_ratio(); - _active = ratio > 0; - - if (_active) { - assert(!UseG1GC, "G1 should not be using dead space"); - - // We allow some amount of garbage towards the bottom of the space, so - // we don't start compacting before there is a significant gain to be made. - // Occasionally, we want to ensure a full compaction, which is determined - // by the MarkSweepAlwaysCompactCount parameter. - if ((MarkSweep::total_invocations() % MarkSweepAlwaysCompactCount) != 0) { - _allowed_deadspace_words = (space->capacity() * ratio / 100) / HeapWordSize; - } else { - _active = false; - } - } - } - - bool insert_deadspace(HeapWord* dead_start, HeapWord* dead_end) { - if (!_active) { - return false; - } - - size_t dead_length = pointer_delta(dead_end, dead_start); - if (_allowed_deadspace_words >= dead_length) { - _allowed_deadspace_words -= dead_length; - CollectedHeap::fill_with_object(dead_start, dead_length); - oop obj = cast_to_oop(dead_start); - obj->set_mark(obj->mark().set_marked()); - - assert(dead_length == obj->size(), "bad filler object size"); - log_develop_trace(gc, compaction)("Inserting object to dead space: " PTR_FORMAT ", " PTR_FORMAT ", " SIZE_FORMAT "b", - p2i(dead_start), p2i(dead_end), dead_length * HeapWordSize); - - return true; - } else { - _active = false; - return false; - } - } -}; - -#ifdef ASSERT -inline void ContiguousSpace::verify_up_to_first_dead(ContiguousSpace* space) { - HeapWord* cur_obj = space->bottom(); - - if (cur_obj < space->_end_of_live && space->_first_dead > cur_obj && !cast_to_oop(cur_obj)->is_gc_marked()) { - // we have a chunk of the space which hasn't moved and we've reinitialized - // the mark word during the previous pass, so we can't use is_gc_marked for - // the traversal. - HeapWord* prev_obj = nullptr; - - while (cur_obj < space->_first_dead) { - size_t size = cast_to_oop(cur_obj)->size(); - assert(!cast_to_oop(cur_obj)->is_gc_marked(), "should be unmarked (special dense prefix handling)"); - prev_obj = cur_obj; - cur_obj += size; - } - } -} -#endif - -inline void ContiguousSpace::clear_empty_region(ContiguousSpace* space) { - // Let's remember if we were empty before we did the compaction. - bool was_empty = space->used_region().is_empty(); - // Reset space after compaction is complete - space->reset_after_compaction(); - // We do this clear, below, since it has overloaded meanings for some - // space subtypes. For example, TenuredSpace's that were - // compacted into will have had their offset table thresholds updated - // continuously, but those that weren't need to have their thresholds - // re-initialized. Also mangles unused area for debugging. - if (space->used_region().is_empty()) { - if (!was_empty) space->clear(SpaceDecorator::Mangle); - } else { - if (ZapUnusedHeapArea) space->mangle_unused_area(); - } +inline void TenuredSpace::update_for_block(HeapWord* start, HeapWord* end) { + _offsets.update_for_block(start, end); } #endif // INCLUDE_SERIALGC diff --git a/src/hotspot/share/gc/shared/vmStructs_gc.hpp b/src/hotspot/share/gc/shared/vmStructs_gc.hpp index 696cdc00dc520..a5d671a3b9295 100644 --- a/src/hotspot/share/gc/shared/vmStructs_gc.hpp +++ b/src/hotspot/share/gc/shared/vmStructs_gc.hpp @@ -99,10 +99,6 @@ nonstatic_field(CollectedHeap, _is_gc_active, bool) \ nonstatic_field(CollectedHeap, _total_collections, unsigned int) \ \ - nonstatic_field(ContiguousSpace, _compaction_top, HeapWord*) \ - nonstatic_field(ContiguousSpace, _first_dead, HeapWord*) \ - nonstatic_field(ContiguousSpace, _end_of_live, HeapWord*) \ - \ nonstatic_field(ContiguousSpace, _top, HeapWord*) \ nonstatic_field(ContiguousSpace, _saved_mark_word, HeapWord*) \ \