Lines Matching full:region
33 // PrimaryEnableRandomOffset is set, each Region actually starts at a random
41 // The 1st Region (for size class 0) holds the TransferBatches. This is a
57 "Group size shouldn't be greater than the region size");
138 RegionInfo *Region = getRegionInfo(I); in init() local
140 initRegion(Region, I, RegionMemMap, Config::getEnableRandomOffset()); in init()
145 // The binding should be done after region shuffling so that it won't bind in init()
146 // the FLLock from the wrong region. in init()
158 RegionInfo *Region = getRegionInfo(I); in unmapTestOnly() local
160 ScopedLock ML(Region->MMLock); in unmapTestOnly()
161 MemMapT MemMap = Region->MemMapInfo.MemMap; in unmapTestOnly()
165 *Region = {}; in unmapTestOnly()
177 RegionInfo *Region = getRegionInfo(I); in verifyAllBlocksAreReleasedTestOnly() local
178 ScopedLock ML(Region->MMLock); in verifyAllBlocksAreReleasedTestOnly()
179 ScopedLock FL(Region->FLLock); in verifyAllBlocksAreReleasedTestOnly()
182 for (BatchGroupT &BG : Region->FreeListInfo.BlockList) { in verifyAllBlocksAreReleasedTestOnly()
189 DCHECK_EQ(TotalBlocks, Region->MemMapInfo.AllocatedUser / BlockSize); in verifyAllBlocksAreReleasedTestOnly()
190 DCHECK_EQ(Region->FreeListInfo.PushedBlocks, in verifyAllBlocksAreReleasedTestOnly()
191 Region->FreeListInfo.PoppedBlocks); in verifyAllBlocksAreReleasedTestOnly()
194 RegionInfo *Region = getRegionInfo(SizeClassMap::BatchClassId); in verifyAllBlocksAreReleasedTestOnly() local
195 ScopedLock ML(Region->MMLock); in verifyAllBlocksAreReleasedTestOnly()
196 ScopedLock FL(Region->FLLock); in verifyAllBlocksAreReleasedTestOnly()
199 for (BatchGroupT &BG : Region->FreeListInfo.BlockList) { in verifyAllBlocksAreReleasedTestOnly()
210 Region->MemMapInfo.AllocatedUser / BlockSize); in verifyAllBlocksAreReleasedTestOnly()
211 DCHECK_GE(Region->FreeListInfo.PoppedBlocks, in verifyAllBlocksAreReleasedTestOnly()
212 Region->FreeListInfo.PushedBlocks); in verifyAllBlocksAreReleasedTestOnly()
214 Region->FreeListInfo.PoppedBlocks - Region->FreeListInfo.PushedBlocks; in verifyAllBlocksAreReleasedTestOnly()
221 RegionInfo *Region = getRegionInfo(ClassId); in popBlocks() local
225 ScopedLock L(Region->FLLock); in popBlocks()
226 PopCount = popBlocksImpl(C, ClassId, Region, ToArray, MaxBlockCount); in popBlocks()
234 PopCount = popBlocksWithCV(C, ClassId, Region, ToArray, MaxBlockCount, in popBlocks()
238 // When two threads compete for `Region->MMLock`, we only want one of in popBlocks()
241 ScopedLock ML(Region->MMLock); in popBlocks()
243 ScopedLock FL(Region->FLLock); in popBlocks()
244 PopCount = popBlocksImpl(C, ClassId, Region, ToArray, MaxBlockCount); in popBlocks()
249 const bool RegionIsExhausted = Region->Exhausted; in popBlocks()
251 PopCount = populateFreeListAndPopBlocks(C, ClassId, Region, ToArray, in popBlocks()
254 ReportRegionExhausted = !RegionIsExhausted && Region->Exhausted; in popBlocks()
277 RegionInfo *Region = getRegionInfo(ClassId); in pushBlocks() local
279 ScopedLock L(Region->FLLock); in pushBlocks()
280 pushBatchClassBlocks(Region, Array, Size); in pushBlocks()
282 Region->FLLockCV.notifyAll(Region->FLLock); in pushBlocks()
307 ScopedLock L(Region->FLLock); in pushBlocks()
308 pushBlocksImpl(C, ClassId, Region, Array, Size, SameGroup); in pushBlocks()
310 Region->FLLockCV.notifyAll(Region->FLLock); in pushBlocks()
341 RegionInfo *Region = getRegionInfo(I); in iterateOverBlocks() local
343 // SizeClassAllocator64. We may consider locking each region on demand in iterateOverBlocks()
345 Region->FLLock.assertHeld(); in iterateOverBlocks()
346 Region->MMLock.assertHeld(); in iterateOverBlocks()
348 const uptr From = Region->RegionBeg; in iterateOverBlocks()
349 const uptr To = From + Region->MemMapInfo.AllocatedUser; in iterateOverBlocks()
356 // TODO(kostyak): get the RSS per region. in getStats()
361 RegionInfo *Region = getRegionInfo(I); in getStats() local
363 ScopedLock L(Region->MMLock); in getStats()
364 TotalMapped += Region->MemMapInfo.MappedUser; in getStats()
367 ScopedLock L(Region->FLLock); in getStats()
368 PoppedBlocks += Region->FreeListInfo.PoppedBlocks; in getStats()
369 PushedBlocks += Region->FreeListInfo.PushedBlocks; in getStats()
379 RegionInfo *Region = getRegionInfo(I); in getStats() local
380 ScopedLock L1(Region->MMLock); in getStats()
381 ScopedLock L2(Region->FLLock); in getStats()
382 getStats(Str, I, Region); in getStats()
392 RegionInfo *Region = getRegionInfo(I); in getFragmentationInfo() local
393 ScopedLock L(Region->MMLock); in getFragmentationInfo()
394 getRegionFragmentationInfo(Region, I, Str); in getFragmentationInfo()
411 RegionInfo *Region = getRegionInfo(ClassId); in tryReleaseToOS() local
415 if (Region->MMLock.tryLock()) { in tryReleaseToOS()
416 uptr BytesReleased = releaseToOSMaybe(Region, ClassId, ReleaseType); in tryReleaseToOS()
417 Region->MMLock.unlock(); in tryReleaseToOS()
428 RegionInfo *Region = getRegionInfo(I); in releaseToOS() local
429 ScopedLock L(Region->MMLock); in releaseToOS()
430 TotalReleasedBytes += releaseToOSMaybe(Region, I, ReleaseType); in releaseToOS()
569 RegionInfo *Region = getRegionInfo(ClassId); in getRegionBaseByClassId() local
570 Region->MMLock.assertHeld(); in getRegionBaseByClassId()
573 !Region->MemMapInfo.MemMap.isAllocated()) { in getRegionBaseByClassId()
576 return Region->MemMapInfo.MemMap.getBase(); in getRegionBaseByClassId()
606 ALWAYS_INLINE void initRegion(RegionInfo *Region, uptr ClassId, in initRegion() argument
608 REQUIRES(Region->MMLock) { in initRegion()
609 DCHECK(!Region->MemMapInfo.MemMap.isAllocated()); in initRegion()
614 Region->MemMapInfo.MemMap = MemMap; in initRegion()
616 Region->RegionBeg = MemMap.getBase(); in initRegion()
618 Region->RegionBeg += in initRegion()
619 (getRandomModN(&Region->RandState, 16) + 1) * PageSize; in initRegion()
625 Region->TryReleaseThreshold = PageSize * SmallerBlockReleasePageDelta; in initRegion()
627 Region->TryReleaseThreshold = PageSize; in initRegion()
630 void pushBatchClassBlocks(RegionInfo *Region, CompactPtrT *Array, u32 Size) in pushBatchClassBlocks() argument
631 REQUIRES(Region->FLLock) { in pushBatchClassBlocks()
632 DCHECK_EQ(Region, getRegionInfo(SizeClassMap::BatchClassId)); in pushBatchClassBlocks()
669 Region->FreeListInfo.PushedBlocks += Size; in pushBatchClassBlocks()
670 BatchGroupT *BG = Region->FreeListInfo.BlockList.front(); in pushBatchClassBlocks()
688 Region->FreeListInfo.BlockList.push_front(BG); in pushBatchClassBlocks()
757 void pushBlocksImpl(CacheT *C, uptr ClassId, RegionInfo *Region,
759 REQUIRES(Region->FLLock) {
805 Region->FreeListInfo.PushedBlocks += Size;
806 BatchGroupT *Cur = Region->FreeListInfo.BlockList.front();
822 Region->FreeListInfo.BlockList.push_front(Cur);
824 Region->FreeListInfo.BlockList.insert(Prev, Cur);
855 Region->FreeListInfo.BlockList.insert(Prev, Cur);
867 u16 popBlocksWithCV(CacheT *C, uptr ClassId, RegionInfo *Region, in popBlocksWithCV() argument
879 ScopedLock FL(Region->FLLock); in popBlocksWithCV()
880 if (!Region->isPopulatingFreeList) { in popBlocksWithCV()
881 Region->isPopulatingFreeList = true; in popBlocksWithCV()
887 ScopedLock ML(Region->MMLock); in popBlocksWithCV()
889 const bool RegionIsExhausted = Region->Exhausted; in popBlocksWithCV()
891 PopCount = populateFreeListAndPopBlocks(C, ClassId, Region, ToArray, in popBlocksWithCV()
894 ReportRegionExhausted = !RegionIsExhausted && Region->Exhausted; in popBlocksWithCV()
900 // `Region->isPopulatingFreeList` to false so the threads about to in popBlocksWithCV()
902 ScopedLock FL(Region->FLLock); in popBlocksWithCV()
903 Region->isPopulatingFreeList = false; in popBlocksWithCV()
904 Region->FLLockCV.notifyAll(Region->FLLock); in popBlocksWithCV()
912 // 2. Region->isPopulatingFreeList == true, i.e, someone is still doing in popBlocksWithCV()
916 // Region->isPopulatingFreeList == false because all the new populated in popBlocksWithCV()
919 ScopedLock FL(Region->FLLock); in popBlocksWithCV()
920 PopCount = popBlocksImpl(C, ClassId, Region, ToArray, MaxBlockCount); in popBlocksWithCV()
924 if (!Region->isPopulatingFreeList) in popBlocksWithCV()
932 Region->FLLockCV.wait(Region->FLLock); in popBlocksWithCV()
934 PopCount = popBlocksImpl(C, ClassId, Region, ToArray, MaxBlockCount); in popBlocksWithCV()
942 u16 popBlocksImpl(CacheT *C, uptr ClassId, RegionInfo *Region, in popBlocksImpl() argument
944 REQUIRES(Region->FLLock) { in popBlocksImpl()
945 if (Region->FreeListInfo.BlockList.empty()) in popBlocksImpl()
949 Region->FreeListInfo.BlockList.front()->Batches; in popBlocksImpl()
953 BatchGroupT *BG = Region->FreeListInfo.BlockList.front(); in popBlocksImpl()
954 Region->FreeListInfo.BlockList.pop_front(); in popBlocksImpl()
961 Region->FreeListInfo.PoppedBlocks += 1; in popBlocksImpl()
995 BatchGroupT *BG = Region->FreeListInfo.BlockList.front(); in popBlocksImpl()
996 Region->FreeListInfo.BlockList.pop_front(); in popBlocksImpl()
1008 Region->FreeListInfo.PoppedBlocks += PopCount; in popBlocksImpl()
1014 RegionInfo *Region, in populateFreeListAndPopBlocks() argument
1017 REQUIRES(Region->MMLock) EXCLUDES(Region->FLLock) { in populateFreeListAndPopBlocks()
1019 !Region->MemMapInfo.MemMap.isAllocated()) { in populateFreeListAndPopBlocks()
1028 initRegion(Region, ClassId, in populateFreeListAndPopBlocks()
1034 DCHECK(Region->MemMapInfo.MemMap.isAllocated()); in populateFreeListAndPopBlocks()
1037 const uptr RegionBeg = Region->RegionBeg; in populateFreeListAndPopBlocks()
1038 const uptr MappedUser = Region->MemMapInfo.MappedUser; in populateFreeListAndPopBlocks()
1040 Region->MemMapInfo.AllocatedUser + MaxCount * Size; in populateFreeListAndPopBlocks()
1048 Region->Exhausted = true; in populateFreeListAndPopBlocks()
1052 if (UNLIKELY(!Region->MemMapInfo.MemMap.remap( in populateFreeListAndPopBlocks()
1059 Region->MemMapInfo.MappedUser += MapSize; in populateFreeListAndPopBlocks()
1065 static_cast<u32>((Region->MemMapInfo.MappedUser - in populateFreeListAndPopBlocks()
1066 Region->MemMapInfo.AllocatedUser) / in populateFreeListAndPopBlocks()
1076 uptr P = RegionBeg + Region->MemMapInfo.AllocatedUser; in populateFreeListAndPopBlocks()
1080 ScopedLock L(Region->FLLock); in populateFreeListAndPopBlocks()
1087 shuffle(ShuffleArray + I - N, N, &Region->RandState); in populateFreeListAndPopBlocks()
1088 pushBlocksImpl(C, ClassId, Region, ShuffleArray + I - N, N, in populateFreeListAndPopBlocks()
1097 shuffle(ShuffleArray + NumberOfBlocks - N, N, &Region->RandState); in populateFreeListAndPopBlocks()
1098 pushBlocksImpl(C, ClassId, Region, &ShuffleArray[NumberOfBlocks - N], N, in populateFreeListAndPopBlocks()
1101 pushBatchClassBlocks(Region, ShuffleArray, NumberOfBlocks); in populateFreeListAndPopBlocks()
1105 popBlocksImpl(C, ClassId, Region, ToArray, MaxBlockCount); in populateFreeListAndPopBlocks()
1112 Region->FreeListInfo.PushedBlocks -= NumberOfBlocks; in populateFreeListAndPopBlocks()
1116 Region->MemMapInfo.AllocatedUser += AllocatedUser; in populateFreeListAndPopBlocks()
1121 void getStats(ScopedString *Str, uptr ClassId, RegionInfo *Region) in getStats() argument
1122 REQUIRES(Region->MMLock, Region->FLLock) { in getStats()
1123 if (Region->MemMapInfo.MappedUser == 0) in getStats()
1127 Region->FreeListInfo.PoppedBlocks - Region->FreeListInfo.PushedBlocks; in getStats()
1129 Region->MemMapInfo.AllocatedUser - InUseBlocks * BlockSize; in getStats()
1132 Region->ReleaseInfo.BytesInFreeListAtLastCheckpoint) { in getStats()
1134 BytesInFreeList - Region->ReleaseInfo.BytesInFreeListAtLastCheckpoint; in getStats()
1136 const uptr TotalChunks = Region->MemMapInfo.AllocatedUser / BlockSize; in getStats()
1140 "released: %6zuK latest pushed bytes: %6zuK region: 0x%zx (0x%zx)\n", in getStats()
1141 Region->Exhausted ? "E" : " ", ClassId, getSizeByClassId(ClassId), in getStats()
1142 Region->MemMapInfo.MappedUser >> 10, Region->FreeListInfo.PoppedBlocks, in getStats()
1143 Region->FreeListInfo.PushedBlocks, InUseBlocks, TotalChunks, in getStats()
1144 Region->ReleaseInfo.RangesReleased, in getStats()
1145 Region->ReleaseInfo.LastReleasedBytes >> 10, in getStats()
1146 RegionPushedBytesDelta >> 10, Region->RegionBeg, in getStats()
1150 void getRegionFragmentationInfo(RegionInfo *Region, uptr ClassId, in getRegionFragmentationInfo() argument
1151 ScopedString *Str) REQUIRES(Region->MMLock) { in getRegionFragmentationInfo()
1154 Region->MemMapInfo.AllocatedUser + Region->RegionBeg; in getRegionFragmentationInfo()
1158 ScopedLock L(Region->FLLock); in getRegionFragmentationInfo()
1159 GroupsToRelease = Region->FreeListInfo.BlockList; in getRegionFragmentationInfo()
1160 Region->FreeListInfo.BlockList.clear(); in getRegionFragmentationInfo()
1166 markFreeBlocks(Region, BlockSize, AllocatedUserEnd, in getRegionFragmentationInfo()
1171 mergeGroupsToReleaseBack(Region, GroupsToRelease); in getRegionFragmentationInfo()
1174 ScopedLock L(Region->FLLock); in getRegionFragmentationInfo()
1176 const uptr TotalBlocks = Region->MemMapInfo.AllocatedUser / BlockSize; in getRegionFragmentationInfo()
1178 Region->FreeListInfo.PoppedBlocks - Region->FreeListInfo.PushedBlocks; in getRegionFragmentationInfo()
1180 roundUp(Region->MemMapInfo.AllocatedUser, PageSize) / PageSize; in getRegionFragmentationInfo()
1196 NOINLINE uptr releaseToOSMaybe(RegionInfo *Region, uptr ClassId,
1198 REQUIRES(Region->MMLock) EXCLUDES(Region->FLLock) {
1202 Region->MemMapInfo.AllocatedUser + Region->RegionBeg;
1206 ScopedLock L(Region->FLLock);
1208 BytesInFreeList = Region->MemMapInfo.AllocatedUser -
1209 (Region->FreeListInfo.PoppedBlocks -
1210 Region->FreeListInfo.PushedBlocks) *
1220 !hasChanceToReleasePages(Region, BlockSize, BytesInFreeList,
1230 GroupsToRelease = Region->FreeListInfo.BlockList;
1231 Region->FreeListInfo.BlockList.clear();
1234 collectGroupsToRelease(Region, BlockSize, AllocatedUserEnd,
1241 // Note that we have extracted the `GroupsToRelease` from region freelist.
1242 // It's safe to let pushBlocks()/popBlocks() access the remaining region
1252 markFreeBlocks(Region, BlockSize, AllocatedUserEnd,
1255 mergeGroupsToReleaseBack(Region, GroupsToRelease);
1262 RegionReleaseRecorder<MemMapT> Recorder(&Region->MemMapInfo.MemMap,
1263 Region->RegionBeg,
1268 Region->ReleaseInfo.BytesInFreeListAtLastCheckpoint = BytesInFreeList;
1269 Region->ReleaseInfo.RangesReleased += Recorder.getReleasedRangesCount();
1270 Region->ReleaseInfo.LastReleasedBytes = Recorder.getReleasedBytes();
1272 Region->ReleaseInfo.LastReleaseAtNs = getMonotonicTimeFast();
1277 mergeGroupsToReleaseBack(Region, GroupsToRelease);
1282 bool hasChanceToReleasePages(RegionInfo *Region, uptr BlockSize, in hasChanceToReleasePages() argument
1284 REQUIRES(Region->MMLock, Region->FLLock) { in hasChanceToReleasePages()
1285 DCHECK_GE(Region->FreeListInfo.PoppedBlocks, in hasChanceToReleasePages()
1286 Region->FreeListInfo.PushedBlocks); in hasChanceToReleasePages()
1291 // following is the region usage, in hasChanceToReleasePages()
1306 Region->ReleaseInfo.BytesInFreeListAtLastCheckpoint) { in hasChanceToReleasePages()
1307 Region->ReleaseInfo.BytesInFreeListAtLastCheckpoint = BytesInFreeList; in hasChanceToReleasePages()
1311 BytesInFreeList - Region->ReleaseInfo.BytesInFreeListAtLastCheckpoint; in hasChanceToReleasePages()
1319 if (RegionPushedBytesDelta < Region->TryReleaseThreshold) in hasChanceToReleasePages()
1334 if (Region->ReleaseInfo.LastReleaseAtNs + in hasChanceToReleasePages()
1347 collectGroupsToRelease(RegionInfo *Region, const uptr BlockSize, in collectGroupsToRelease() argument
1349 REQUIRES(Region->MMLock, Region->FLLock) { in collectGroupsToRelease()
1355 // release threshold as the next Region::TryReleaseThreshold(). Note that if in collectGroupsToRelease()
1361 for (BatchGroupT *BG = Region->FreeListInfo.BlockList.front(), in collectGroupsToRelease()
1379 DCHECK_LE(Region->RegionBeg, BatchGroupBase); in collectGroupsToRelease()
1381 DCHECK_EQ((Region->RegionBeg - BatchGroupBase) % GroupSize, 0U); in collectGroupsToRelease()
1493 Region->FreeListInfo.BlockList.extract(Prev, Cur); in collectGroupsToRelease()
1495 Region->FreeListInfo.BlockList.pop_front(); in collectGroupsToRelease()
1506 Region->TryReleaseThreshold = MinDistToThreshold; in collectGroupsToRelease()
1513 markFreeBlocks(RegionInfo *Region, const uptr BlockSize, in markFreeBlocks() argument
1516 REQUIRES(Region->MMLock) EXCLUDES(Region->FLLock) { in markFreeBlocks()
1532 roundUpSlow(LastGroupEnd - Region->RegionBeg, BlockSize) + in markFreeBlocks()
1533 Region->RegionBeg; in markFreeBlocks()
1535 const uptr ReleaseOffset = ReleaseBase - Region->RegionBeg; in markFreeBlocks()
1555 (BatchGroupUsedEnd - Region->RegionBeg) % BlockSize == 0; in markFreeBlocks()
1573 Region->RegionBeg, /*RegionIndex=*/0, in markFreeBlocks()
1574 Region->MemMapInfo.AllocatedUser); in markFreeBlocks()
1581 BG.Batches, DecompactPtr, Region->RegionBeg, /*RegionIndex=*/0, in markFreeBlocks()
1582 Region->MemMapInfo.AllocatedUser, MayContainLastBlockInRegion); in markFreeBlocks()
1591 void mergeGroupsToReleaseBack(RegionInfo *Region, in mergeGroupsToReleaseBack() argument
1593 REQUIRES(Region->MMLock) EXCLUDES(Region->FLLock) { in mergeGroupsToReleaseBack()
1594 ScopedLock L(Region->FLLock); in mergeGroupsToReleaseBack()
1608 DCHECK_NE(BatchClassRegion, Region); in mergeGroupsToReleaseBack()
1610 // Merge GroupsToRelease back to the Region::FreeListInfo.BlockList. Note in mergeGroupsToReleaseBack()
1611 // that both `Region->FreeListInfo.BlockList` and `GroupsToRelease` are in mergeGroupsToReleaseBack()
1613 for (BatchGroupT *BG = Region->FreeListInfo.BlockList.front(), in mergeGroupsToReleaseBack()
1618 Region->FreeListInfo.BlockList.append_back(&GroupsToRelease); in mergeGroupsToReleaseBack()
1702 Region->FreeListInfo.BlockList.push_front(Cur); in mergeGroupsToReleaseBack()
1704 Region->FreeListInfo.BlockList.insert(Prev, Cur); in mergeGroupsToReleaseBack()
1717 BatchGroupT *Prev = Region->FreeListInfo.BlockList.front(); in mergeGroupsToReleaseBack()
1725 Region->FLLockCV.notifyAll(Region->FLLock); in mergeGroupsToReleaseBack()