Lines Matching full:chunk
14 #include "chunk.h"
69 // Chunk recycling function, returns a quarantined chunk to the backend,
72 Chunk::UnpackedHeader Header; in recycle()
73 Chunk::loadHeader(Allocator.Cookie, Ptr, &Header); in recycle()
74 if (UNLIKELY(Header.State != Chunk::State::Quarantined)) in recycle()
77 Header.State = Chunk::State::Available; in recycle()
78 Chunk::storeHeader(Allocator.Cookie, Ptr, &Header); in recycle()
91 sizeof(QuarantineBatch) + Chunk::getHeaderSize()); in allocate()
98 Chunk::getHeaderSize()); in allocate()
99 Chunk::UnpackedHeader Header = {}; in allocate()
100 Header.ClassId = QuarantineClassId & Chunk::ClassIdMask; in allocate()
102 Header.State = Chunk::State::Allocated; in allocate()
103 Chunk::storeHeader(Allocator.Cookie, Ptr, &Header); in allocate()
105 // Reset tag to 0 as this chunk may have been previously used for a tagged in allocate()
117 sizeof(QuarantineBatch) + Chunk::getHeaderSize()); in deallocate()
118 Chunk::UnpackedHeader Header; in deallocate()
119 Chunk::loadHeader(Allocator.Cookie, Ptr, &Header); in deallocate()
121 if (UNLIKELY(Header.State != Chunk::State::Allocated)) in deallocate()
127 Header.State = Chunk::State::Available; in deallocate()
128 Chunk::storeHeader(Allocator.Cookie, Ptr, &Header); in deallocate()
131 Chunk::getHeaderSize())); in deallocate()
326 // If a chunk's tag is odd, we want the tags of the surrounding blocks to be in computeOddEvenMaskForPointerMaybe()
334 NOINLINE void *allocate(uptr Size, Chunk::Origin Origin,
372 ((Alignment > MinAlignment) ? Alignment : Chunk::getHeaderSize());
414 reinterpret_cast<uptr>(Block) + Chunk::getHeaderSize(), Alignment);
427 NOINLINE void deallocate(void *Ptr, Chunk::Origin Origin, uptr DeleteSize = 0,
457 Chunk::UnpackedHeader Header;
458 Chunk::loadHeader(Cookie, Ptr, &Header);
460 if (UNLIKELY(Header.State != Chunk::State::Allocated))
467 if (Header.OriginOrWasZeroed != Chunk::Origin::Memalign ||
468 Origin != Chunk::Origin::Malloc)
500 void *NewPtr = allocate(NewSize, Chunk::Origin::Malloc, Alignment);
518 Chunk::UnpackedHeader Header;
519 Chunk::loadHeader(Cookie, OldPtr, &Header);
521 if (UNLIKELY(Header.State != Chunk::State::Allocated))
528 if (UNLIKELY(Header.OriginOrWasZeroed != Chunk::Origin::Malloc))
531 Chunk::Origin::Malloc);
547 // If the new chunk still fits in the previously allocated block (with a
548 // reasonable delta), we just keep the old block, and update the chunk
569 Chunk::SizeOrUnusedBytesMask;
570 Chunk::storeHeader(Cookie, OldPtr, &Header);
586 // allocators will allocate an even larger chunk (by a fixed factor) to
589 void *NewPtr = allocate(NewSize, Chunk::Origin::Malloc, Alignment);
681 uptr Chunk; in iterateOverChunks() local
682 Chunk::UnpackedHeader Header; in iterateOverChunks()
684 // A chunk header can either have a zero tag (tagged primary) or the in iterateOverChunks()
688 if (!getChunkFromBlock(Block, &Chunk, &Header) && in iterateOverChunks()
689 !getChunkFromBlock(addHeaderTag(Block), &Chunk, &Header)) in iterateOverChunks()
692 if (!getChunkFromBlock(addHeaderTag(Block), &Chunk, &Header)) in iterateOverChunks()
695 if (Header.State == Chunk::State::Allocated) { in iterateOverChunks()
696 uptr TaggedChunk = Chunk; in iterateOverChunks()
700 TaggedChunk = loadTag(Chunk); in iterateOverChunks()
701 Callback(TaggedChunk, getSize(reinterpret_cast<void *>(Chunk), &Header), in iterateOverChunks()
725 // any particular chunk is cut in half. Therefore we use this tuning in setOption()
744 // Return the usable size for a given chunk. Technically we lie, as we just
745 // report the actual size of a chunk. This is done to counteract code actively
746 // writing past the end of a chunk (like sqlite3) when the usable size allows
747 // for it, which then forces realloc to copy the usable size of a chunk as
765 Chunk::UnpackedHeader Header; in getAllocSize()
766 Chunk::loadHeader(Cookie, Ptr, &Header); in getAllocSize()
768 // Getting the alloc size of a chunk only makes sense if it's allocated. in getAllocSize()
769 if (UNLIKELY(Header.State != Chunk::State::Allocated)) in getAllocSize()
782 // A corrupted chunk will not be reported as owned, which is WAI.
792 Chunk::UnpackedHeader Header; in isOwned()
793 return Chunk::isValid(Cookie, Ptr, &Header) && in isOwned()
794 Header.State == Chunk::State::Allocated; in isOwned()
943 static_assert(MinAlignment >= sizeof(Chunk::PackedHeader),
944 "Minimal alignment must at least cover a chunk header.");
952 // inline with a chunk that is relevant to diagnosing memory tag faults, where
955 // which corresponds to 8 bytes before the user memory, because the chunk
1021 Chunk::UnpackedHeader Header = {}; in performSanityChecks()
1025 (MaxPrimaryAlignment - Chunk::getHeaderSize()) >> MinAlignmentLog; in performSanityChecks()
1026 Header.Offset = MaxOffset & Chunk::OffsetMask; in performSanityChecks()
1047 Chunk::UnpackedHeader *Header) { in getBlockBegin()
1049 reinterpret_cast<uptr>(Ptr) - Chunk::getHeaderSize() - in getBlockBegin()
1053 // Return the size of a chunk as requested during its allocation.
1054 inline uptr getSize(const void *Ptr, Chunk::UnpackedHeader *Header) { in getSize()
1064 ALWAYS_INLINE void *initChunk(const uptr ClassId, const Chunk::Origin Origin, in initChunk()
1070 reinterpret_cast<uptr>(Block) + Chunk::getHeaderSize(); in initChunk()
1082 Chunk::UnpackedHeader Header = {}; in initChunk()
1088 // the chunk iteration function that can be used in debugging situations. in initChunk()
1089 // It is the only situation where we have to locate the start of a chunk in initChunk()
1093 Header.Offset = (Offset >> MinAlignmentLog) & Chunk::OffsetMask; in initChunk()
1096 Header.ClassId = ClassId & Chunk::ClassIdMask; in initChunk()
1097 Header.State = Chunk::State::Allocated; in initChunk()
1098 Header.OriginOrWasZeroed = Origin & Chunk::OriginMask; in initChunk()
1099 Header.SizeOrUnusedBytes = SizeOrUnusedBytes & Chunk::SizeOrUnusedBytesMask; in initChunk()
1100 Chunk::storeHeader(Cookie, reinterpret_cast<void *>(addHeaderTag(UserPtr)), in initChunk()
1107 initChunkWithMemoryTagging(const uptr ClassId, const Chunk::Origin Origin, in initChunkWithMemoryTagging()
1116 reinterpret_cast<uptr>(Block) + Chunk::getHeaderSize(); in initChunkWithMemoryTagging()
1122 // Init the primary chunk. in initChunkWithMemoryTagging()
1136 Chunk::UnpackedHeader Header; in initChunkWithMemoryTagging()
1146 // the reclaimed portions if necessary. In the case where the chunk is in initChunkWithMemoryTagging()
1147 // fully reclaimed, the chunk's header will be zero, which will trigger in initChunkWithMemoryTagging()
1149 // chunk from scratch. There are three possibilities for partial in initChunkWithMemoryTagging()
1161 // of the chunk. If it is zero, it means that either all data was in initChunkWithMemoryTagging()
1162 // reclaimed (since we never use zero as the chunk tag), or that the in initChunkWithMemoryTagging()
1164 // a new chunk from scratch. in initChunkWithMemoryTagging()
1167 // chunk) and loading the tag of its first granule. If it is zero, it in initChunkWithMemoryTagging()
1196 // the chunk data. in initChunkWithMemoryTagging()
1206 // Init the secondary chunk. in initChunkWithMemoryTagging()
1214 Chunk::UnpackedHeader Header = {}; in initChunkWithMemoryTagging()
1220 // the chunk iteration function that can be used in debugging situations. in initChunkWithMemoryTagging()
1221 // It is the only situation where we have to locate the start of a chunk in initChunkWithMemoryTagging()
1225 Header.Offset = (Offset >> MinAlignmentLog) & Chunk::OffsetMask; in initChunkWithMemoryTagging()
1228 Header.ClassId = ClassId & Chunk::ClassIdMask; in initChunkWithMemoryTagging()
1229 Header.State = Chunk::State::Allocated; in initChunkWithMemoryTagging()
1230 Header.OriginOrWasZeroed = Origin & Chunk::OriginMask; in initChunkWithMemoryTagging()
1231 Header.SizeOrUnusedBytes = SizeOrUnusedBytes & Chunk::SizeOrUnusedBytesMask; in initChunkWithMemoryTagging()
1232 Chunk::storeHeader(Cookie, Ptr, &Header); in initChunkWithMemoryTagging()
1238 Chunk::UnpackedHeader *Header, in quarantineOrDeallocateChunk()
1241 // If the quarantine is disabled, the actual size of a chunk is 0 or larger in quarantineOrDeallocateChunk()
1242 // than the maximum allowed, we return a chunk directly to the backend. in quarantineOrDeallocateChunk()
1248 Header->State = Chunk::State::Available; in quarantineOrDeallocateChunk()
1250 Header->State = Chunk::State::Quarantined; in quarantineOrDeallocateChunk()
1265 Chunk::storeHeader(Cookie, Ptr, Header); in quarantineOrDeallocateChunk()
1292 Chunk::UnpackedHeader *Header, const uptr Size, in retagBlock()
1319 bool getChunkFromBlock(uptr Block, uptr *Chunk, in getChunkFromBlock() argument
1320 Chunk::UnpackedHeader *Header) { in getChunkFromBlock()
1321 *Chunk = in getChunkFromBlock()
1323 return Chunk::isValid(Cookie, reinterpret_cast<void *>(*Chunk), Header); in getChunkFromBlock()
1330 return Offset + Chunk::getHeaderSize(); in getChunkOffsetFromBlock()
1344 // address tag in the first byte of the chunk.
1357 // Prepare the granule before the chunk to store the chunk header by setting in prepareTaggedChunk()
1359 // chunk holding a low alignment allocation is reused for a higher alignment in prepareTaggedChunk()
1360 // allocation, the chunk may already have a non-zero tag from the previous in prepareTaggedChunk()
1487 Chunk::UnpackedHeader *Header, const u32 **Data, in getInlineErrorInfo()
1499 *Header = *reinterpret_cast<const Chunk::UnpackedHeader *>( in getInlineErrorInfo()
1500 ChunkBegin - Chunk::getHeaderSize()); in getInlineErrorInfo()
1504 // the chunk, see storeEndMarker(). in getInlineErrorInfo()
1519 Chunk::UnpackedHeader Header; in getInlineErrorInfo()
1523 Header.State != Chunk::State::Allocated || Tag != FaultAddrTag) in getInlineErrorInfo()