Home
last modified time | relevance | path

Searched full:unlikely (Results 1 – 25 of 1087) sorted by relevance

12345678910>>...44

/freebsd/contrib/jemalloc/include/jemalloc/internal/
H A Dtcache_inlines.h54 if (unlikely(!tcache_success)) { in tcache_alloc_small()
57 if (unlikely(arena == NULL)) { in tcache_alloc_small()
60 if (unlikely(tcache_small_bin_disabled(binind, bin))) { in tcache_alloc_small()
76 if (unlikely(zero)) { in tcache_alloc_small()
97 if (unlikely(!tcache_success)) { in tcache_alloc_small()
103 if (unlikely(arena == NULL)) {
114 if (unlikely(zero)) { in tcache_alloc_large()
135 * Not marking the branch unlikely because this is past free_fastpath() in tcache_alloc_large()
149 if (unlikely(!cache_bin_dalloc_easy(bin, ptr))) { in tcache_alloc_large()
150 if (unlikely(tcache_small_bin_disable in tcache_alloc_large()
[all...]
H A Dprof_inlines_b.h
H A Darena_inlines_b.h31 if (unlikely(size >= oversize_threshold)) { in arena_choose_maybe_huge()
56 } else if (unlikely(!(is_slab = alloc_ctx->slab))) {
61 if (unlikely(!is_slab)) { in arena_prof_tctx_set()
84 if (unlikely(!edata_slab_get(edata))) { in arena_prof_tctx_reset()
88 if (unlikely(!alloc_ctx->slab)) { in arena_prof_alloc_time_get()
119 if (unlikely(tsdn_null(tsdn))) { in arena_decay_ticks()
134 if (unlikely(ticker_geom_ticks(decay_ticker, prng_state, nticks))) { in arena_decay_tick()
224 if (unlikely(edata == NULL || in arena_vsalloc()
233 if (unlikely(input_size != edata_usize_get(edata))) { in arena_dalloc_large_no_tcache()
244 if (config_prof && unlikely(szin in arena_dalloc_no_tcache()
[all...]
/freebsd/sys/dev/mlx5/mlx5_en/
H A Dmlx5_en_tx.c186 if (unlikely(mb->m_len < ETHER_HDR_LEN)) { in mlx5e_get_l2_header_size()
189 if (unlikely(mb->m_len < (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN))) in mlx5e_get_l2_header_size()
248 if (unlikely(mb->m_len < ETHER_HDR_LEN)) in mlx5e_get_full_header_size()
251 if (unlikely(mb->m_len < ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN)) in mlx5e_get_full_header_size()
263 if (unlikely(mb->m_len < eth_hdr_len + sizeof(*ip))) in mlx5e_get_full_header_size()
281 if (unlikely(mb->m_len < eth_hdr_len + sizeof(*ip6))) in mlx5e_get_full_header_size()
299 if (unlikely(mb->m_len < eth_hdr_len + sizeof(*th))) { in mlx5e_get_full_header_size()
301 if (unlikely(mb->m_len != eth_hdr_len || in mlx5e_get_full_header_size()
316 if (unlikely(mb->m_pkthdr.len < eth_hdr_len)) in mlx5e_get_full_header_size()
334 if (unlikely(m in mlx5e_parse_mbuf_chain()
[all...]
/freebsd/sys/contrib/ena-com/
H A Dena_com.c118 if (unlikely((addr & GENMASK_ULL(ena_dev->dma_addr_bits - 1, 0)) != addr)) { in ena_com_mem_addr_set()
138 if (unlikely(!sq->entries)) { in ena_com_admin_init_sq()
161 if (unlikely(!cq->entries)) { in ena_com_admin_init_cq()
186 if (unlikely(!aenq->entries)) { in ena_com_admin_init_aenq()
207 if (unlikely(!aenq_handlers)) { in ena_com_admin_init_aenq()
228 if (unlikely(command_id >= admin_queue->q_depth)) { in get_comp_ctxt()
235 if (unlikely(!admin_queue->comp_ctx)) { in get_comp_ctxt()
241 if (unlikely(admin_queue->comp_ctx[command_id].occupied && capture)) { in get_comp_ctxt()
272 if (unlikely(cnt >= admin_queue->q_depth)) { in __ena_com_submit_admin_cmd()
287 if (unlikely(!comp_ctx)) in __ena_com_submit_admin_cmd()
[all …]
H A Dena_eth_com.c87 if (unlikely(!io_sq->entries_in_tx_burst_left)) { in ena_com_write_bounce_buffer_to_dev()
113 if (unlikely((io_sq->tail & (io_sq->q_depth - 1)) == 0)) in ena_com_write_bounce_buffer_to_dev()
128 if (unlikely(io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST)) in ena_com_write_header_to_bounce()
134 if (unlikely((header_offset + header_len) > llq_info->desc_list_entry_size)) { in ena_com_write_header_to_bounce()
140 if (unlikely(!bounce_buffer)) { in ena_com_write_header_to_bounce()
159 if (unlikely(!bounce_buffer)) { in get_sq_desc_llq()
178 if (unlikely(io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST)) in ena_com_close_bounce_buffer()
185 if (unlikely(rc)) { in ena_com_close_bounce_buffer()
219 if (unlikely(rc)) { in ena_com_sq_update_llq_tail()
231 if (unlikely(llq_info->desc_stride_ctrl == ENA_ADMIN_SINGLE_DESC_PER_ENTRY)) in ena_com_sq_update_llq_tail()
[all …]
/freebsd/sys/contrib/dpdk_rte_lpm/
H A Drte_branch_prediction.h28 * Check if a branch is unlikely to be taken.
31 * unlikely to be taken. Example:
33 * if (unlikely(x < 1))
37 #ifndef unlikely
38 #define unlikely(x) __builtin_expect(!!(x), 0) macro
39 #endif /* unlikely */
/freebsd/sys/dev/drm2/ttm/
H A Dttm_bo_vm.c91 if (unlikely(best_bo == NULL)) in ttm_bo_vm_lookup_rb()
94 if (unlikely((best_bo->vm_node->start + best_bo->num_pages) < in ttm_bo_vm_lookup_rb()
126 if (unlikely(ret != 0)) { in ttm_bo_vm_fault()
174 if (unlikely(ret != 0)) { in ttm_bo_vm_fault()
182 if (unlikely(ret != 0)) { in ttm_bo_vm_fault()
187 if (unlikely(ret != 0)) { in ttm_bo_vm_fault()
225 if (unlikely(!m)) { in ttm_bo_vm_fault()
334 if (unlikely(bo == NULL)) { in ttm_bo_mmap_single()
340 if (unlikely(!driver->verify_access)) { in ttm_bo_mmap_single()
345 if (unlikely(ret != 0)) in ttm_bo_mmap_single()
[all …]
/freebsd/sys/contrib/openzfs/include/os/linux/spl/sys/
H A Ddebug.h105 #define unlikely(expr) expect((expr) != 0, 0) macro
112 (void) (unlikely(!(cond)) && \
117 if (unlikely(!(cond))) \
125 if (unlikely(!(_verify3_left OP _verify3_right))) \
135 if (unlikely(!(_verify3_left OP _verify3_right))) \
146 if (unlikely(!(_verify3_left OP _verify3_right))) \
157 if (unlikely(!(_verify3_left OP _verify3_right))) \
167 if (unlikely(!(0 == _verify0_right))) \
175 if (unlikely(!(0 == _verify0_right))) \
192 if (unlikely(!(_verify3_left OP _verify3_right))) \
[all …]
/freebsd/contrib/ofed/libmlx5/
H A Dqp.c77 if (unlikely(!(*size))) in copy_to_scat()
99 if (unlikely(qp->wq_sig)) in mlx5_copy_to_recv_wqe()
139 if (unlikely((void *)(scat + max) > qp->sq.qend)) { in mlx5_copy_to_send_wqe()
251 if (unlikely(src == qp->sq.qend)) in mlx5_bf_copy()
290 if (unlikely(inl > qp->max_inline_data)) in set_data_inl_seg()
293 if (unlikely(wqe + len > qend)) { in set_data_inl_seg()
368 if (unlikely(wr->num_sge < 1)) { in copy_eth_inline_headers()
389 if (unlikely(inl_hdr_size)) { in copy_eth_inline_headers()
402 if (unlikely(wr->sg_list[j].length == inl_hdr_copy_size)) { in copy_eth_inline_headers()
545 if (unlikely((*seg == qend))) in set_bind_wr()
[all …]
/freebsd/sys/contrib/openzfs/include/os/freebsd/spl/sys/
H A Ddebug.h108 (void) (unlikely(!(cond)) && \
113 if (unlikely(!(cond))) \
121 if (unlikely(!(_verify3_left OP _verify3_right))) \
131 if (unlikely(!(_verify3_left OP _verify3_right))) \
142 if (unlikely(!(_verify3_left OP _verify3_right))) \
153 if (unlikely(!(_verify3_left OP _verify3_right))) \
163 if (unlikely(!(0 == _verify0_right))) \
171 if (unlikely(!(0 == _verify0_right))) \
188 if (unlikely(!(_verify3_left OP _verify3_right))) \
199 if (unlikely(!(_verify3_left OP _verify3_right))) \
[all …]
/freebsd/sys/dev/ena/
H A Dena.c218 if (unlikely(dma_space_addr == 0)) in ena_dma_alloc()
233 if (unlikely(error != 0)) { in ena_dma_alloc()
239 if (unlikely(error != 0)) { in ena_dma_alloc()
247 if (unlikely(error != 0)) { in ena_dma_alloc()
256 if (unlikely((error != 0) || (dma->paddr == 0))) { in ena_dma_alloc()
574 if (unlikely(tx_req_id_rc != 0)) { in validate_tx_req_id()
653 if (unlikely(tx_ring->tx_buffer_info == NULL)) in ena_setup_tx_resources()
658 if (unlikely(tx_ring->free_tx_ids == NULL)) in ena_setup_tx_resources()
664 if (unlikely(tx_ring->push_buf_intermediate_buf == NULL)) in ena_setup_tx_resources()
688 if (unlikely(err != 0)) { in ena_setup_tx_resources()
[all …]
H A Dena_rss.c108 if (unlikely(rc != 0)) { in ena_rss_init_default()
121 if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) { in ena_rss_init_default()
139 if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) { in ena_rss_init_default()
145 if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) { in ena_rss_init_default()
169 if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) { in ena_rss_configure()
180 if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) in ena_rss_configure()
185 if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) in ena_rss_configure()
190 if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) in ena_rss_configure()
205 if (unlikely(dc == NULL)) { in ena_rss_init_default_deferred()
216 if (unlikely(rc != 0)) { in ena_rss_init_default_deferred()
H A Dena_datapath.c92 if (unlikely(((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) || in ena_cleanup()
105 if (unlikely(((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) || in ena_cleanup()
146 if (unlikely((if_getdrvflags(adapter->ifp) & IFF_DRV_RUNNING) == 0)) in ena_mq_start()
171 if (unlikely(ret != 0)) { in ena_mq_start()
214 if (unlikely(rc == ENA_COM_TRY_AGAIN)) in ena_get_tx_req_id()
219 if (unlikely(tx_ring->tx_buffer_info[*req_id].mbuf == NULL)) { in ena_get_tx_req_id()
271 if (unlikely(rc != 0)) in ena_tx_cleanup()
296 if (unlikely(--commit == 0)) { in ena_tx_cleanup()
327 if (unlikely(!tx_ring->running && above_thresh)) { in ena_tx_cleanup()
436 if (unlikely(rx_info->mbuf == NULL)) { in ena_rx_mbuf()
[all …]
H A Dena_netmap.c116 if (unlikely(rx_info->netmap_buf_idx != 0)) in ena_netmap_alloc_rx_slot()
330 if (unlikely(!ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, ctx.adapter))) in ena_netmap_txsync()
333 if (unlikely(!ENA_FLAG_ISSET(ENA_FLAG_LINK_UP, ctx.adapter))) in ena_netmap_txsync()
413 if (unlikely(rc != 0)) { in ena_netmap_tx_frame()
433 if (unlikely(rc != 0)) { in ena_netmap_tx_frame()
508 if (unlikely(slot_vaddr == NULL)) in ena_netmap_copy_data()
531 if (unlikely(vaddr == NULL)) { in ena_netmap_map_single_slot()
537 if (unlikely(rc != 0)) { in ena_netmap_map_single_slot()
598 if (unlikely(push_hdr == NULL)) { in ena_netmap_tx_map_slots()
611 if (unlikely(rc)) { in ena_netmap_tx_map_slots()
[all …]
/freebsd/sys/dev/qat/qat_common/
H A Dadf_pfvf_vf_proto.c194 if (unlikely(type != ADF_PF2VF_BLKMSG_RESP_TYPE_DATA)) { in adf_vf2pf_blkmsg_get_byte()
224 if (unlikely(type != ADF_PF2VF_BLKMSG_RESP_TYPE_CRC)) { in adf_vf2pf_blkmsg_get_crc()
264 if (unlikely(type > ADF_VF2PF_LARGE_BLOCK_TYPE_MAX)) { in adf_send_vf2pf_blkmsg_req()
271 if (unlikely(*buffer_len < ADF_PFVF_BLKMSG_HEADER_SIZE)) { in adf_send_vf2pf_blkmsg_req()
281 if (unlikely(ret)) in adf_send_vf2pf_blkmsg_req()
284 if (unlikely(!buffer[ADF_PFVF_BLKMSG_VER_BYTE])) { in adf_send_vf2pf_blkmsg_req()
295 if (unlikely(ret)) in adf_send_vf2pf_blkmsg_req()
298 if (unlikely(!buffer[ADF_PFVF_BLKMSG_LEN_BYTE])) { in adf_send_vf2pf_blkmsg_req()
322 if (unlikely(ret)) in adf_send_vf2pf_blkmsg_req()
327 if (unlikely(ret)) in adf_send_vf2pf_blkmsg_req()
[all …]
/freebsd/contrib/llvm-project/compiler-rt/lib/scudo/standalone/
H A Dmem_map_fuchsia.cpp44 if (UNLIKELY(!Result)) { in getRootVmarBase()
64 if (UNLIKELY(Vmo == ZX_HANDLE_INVALID)) { in getPlaceholderVmo()
67 if (UNLIKELY(Status != ZX_OK)) in getPlaceholderVmo()
76 if (UNLIKELY(OldValue != ZX_HANDLE_INVALID)) { in getPlaceholderVmo()
99 if (UNLIKELY(Status != ZX_OK)) in MemMapFuchsia()
113 if (UNLIKELY(Status != ZX_OK)) { in mapImpl()
128 if (UNLIKELY(Status != ZX_OK)) { in mapImpl()
199 if (UNLIKELY(Status != ZX_OK)) { in remapImpl()
239 if (UNLIKELY(Status != ZX_OK)) { in createImpl()
H A Dcombined.h74 if (UNLIKELY(Header.State != Chunk::State::Quarantined)) in recycle()
94 if (UNLIKELY(!Ptr)) in allocate()
107 if (UNLIKELY(useMemoryTagging<AllocatorConfig>( in allocate()
121 if (UNLIKELY(Header.State != Chunk::State::Allocated)) in deallocate()
150 if (UNLIKELY(!getRandom(&Cookie, sizeof(Cookie)))) in init()
340 if (UNLIKELY(Alignment > MaxAlignment)) {
349 if (UNLIKELY(GuardedAlloc.shouldSample())) {
376 if (UNLIKELY(Size >= MaxAllowedMallocSize)) {
394 if (UNLIKELY(!Block)) {
401 if (UNLIKELY(ClassId == 0)) {
[all …]
/freebsd/contrib/llvm-project/compiler-rt/lib/sanitizer_common/
H A Dsanitizer_allocator_local_cache.h38 if (UNLIKELY(c->count == 0)) { in Allocate()
39 if (UNLIKELY(!Refill(c, allocator, class_id))) in Allocate()
56 if (UNLIKELY(c->count == c->max_count)) in Deallocate()
103 if (UNLIKELY(!allocator->GetFromAllocator(&stats_, class_id, c->chunks, in Refill()
163 if (UNLIKELY(c->count == 0)) { in Allocate()
164 if (UNLIKELY(!Refill(c, allocator, class_id))) in Allocate()
181 if (UNLIKELY(c->count == c->max_count)) in Deallocate()
245 if (UNLIKELY(!b)) in Refill()
262 if (UNLIKELY(!b)) { in Drain()
/freebsd/contrib/llvm-project/compiler-rt/lib/dfsan/
H A Ddfsan_allocator.cpp101 if (UNLIKELY(IsRssLimitExceeded())) { in DFsanAllocate()
117 if (UNLIKELY(!allocated)) { in DFsanAllocate()
175 if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) { in DFsanCalloc()
231 if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) { in dfsan_reallocarray()
248 if (UNLIKELY(CheckForPvallocOverflow(size, PageSize))) { in dfsan_pvalloc()
261 if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(alignment, size))) { in dfsan_aligned_alloc()
272 if (UNLIKELY(!IsPowerOfTwo(alignment))) { in dfsan_memalign()
283 if (UNLIKELY(!CheckPosixMemalignAlignment(alignment))) { in dfsan_posix_memalign()
290 if (UNLIKELY(!ptr)) in dfsan_posix_memalign()
/freebsd/contrib/jemalloc/src/
H A Djemalloc.c261 if (unlikely(opt_utrace)) { \ in a0dalloc()
294 if (unlikely(malloc_init_state == malloc_init_uninitialized)) { in bootstrap_free()
302 if (unlikely(!malloc_initialized()) && malloc_init_hard()) { in arena_set()
315 if (unlikely(malloc_init_a0())) {
346 if (unlikely(size == 0)) { in arena_init_locked()
358 if (unlikely(num_size == 0)) { in arena_new_create_background_thread()
368 if (unlikely(ptr == NULL)) { in arena_init()
2305 if (unlikely(*ind >= SC_NSIZES)) { in malloc_default()
2314 if (bump_empty_aligned_alloc && unlikely(size == 0)) {
2319 if (unlikely(*usiz
[all...]
H A Dlarge.c30 if (unlikely(ausize == 0 || ausize > SC_LARGE_MAXCLASS)) { in large_palloc()
37 if (unlikely(arena == NULL) || (edata = arena_extent_alloc_large(tsdn, in large_palloc()
/freebsd/contrib/llvm-project/compiler-rt/lib/lsan/
H A Dlsan_allocator.cpp96 if (UNLIKELY(IsRssLimitExceeded())) { in Allocate()
102 if (UNLIKELY(!p)) { in Allocate()
116 if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) { in Calloc()
181 if (UNLIKELY(!CheckPosixMemalignAlignment(alignment))) { in lsan_posix_memalign()
187 if (UNLIKELY(!ptr)) in lsan_posix_memalign()
196 if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(alignment, size))) { in lsan_aligned_alloc()
206 if (UNLIKELY(!IsPowerOfTwo(alignment))) { in lsan_memalign()
229 if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) { in lsan_reallocarray()
249 if (UNLIKELY(CheckForPvallocOverflow(size, PageSize))) { in lsan_pvalloc()
/freebsd/contrib/llvm-project/compiler-rt/lib/msan/
H A Dmsan_allocator.cpp186 if (UNLIKELY(size > max_malloc_size)) { in MsanAllocate()
194 if (UNLIKELY(IsRssLimitExceeded())) { in MsanAllocate()
210 if (UNLIKELY(!allocated)) { in MsanAllocate()
294 if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) { in MsanCalloc()
350 if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) { in msan_reallocarray()
366 if (UNLIKELY(CheckForPvallocOverflow(size, PageSize))) { in msan_pvalloc()
379 if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(alignment, size))) { in msan_aligned_alloc()
390 if (UNLIKELY(!IsPowerOfTwo(alignment))) { in msan_memalign()
402 if (UNLIKELY(!CheckPosixMemalignAlignment(alignment))) { in msan_posix_memalign()
409 if (UNLIKELY(!ptr)) in msan_posix_memalign()
/freebsd/contrib/llvm-project/compiler-rt/lib/tsan/rtl/
H A Dtsan_rtl_access.cpp25 if (UNLIKELY(!TraceAcquire(thr, &ev))) in TryTraceMemoryAccess()
62 if (UNLIKELY(!TraceAcquire(thr, &ev))) in TryTraceMemoryAccessRange()
322 if (UNLIKELY(race_mask)) in CheckRaces()
344 if (UNLIKELY(index == 0)) { in CheckRaces()
348 if (UNLIKELY(index == 0)) in CheckRaces()
438 if (UNLIKELY(fast_state.GetIgnoreBit())) in MemoryAccess()
458 if (UNLIKELY(fast_state.GetIgnoreBit())) in MemoryAccess16()
470 if (UNLIKELY(CheckRaces(thr, shadow_mem, cur, shadow, access, typ))) in MemoryAccess16()
495 if (UNLIKELY(fast_state.GetIgnoreBit())) in UnalignedMemoryAccess()
508 if (UNLIKELY(CheckRaces(thr, shadow_mem, cur, shadow, access, typ))) in UnalignedMemoryAccess()
[all …]

12345678910>>...44