From 8d75a75a5e699454a78b8d81a4e0bc58fc4bab04 Mon Sep 17 00:00:00 2001 From: "Dr. David Alan Gilbert (Dragon)" Date: Wed, 22 Jul 2026 21:51:33 +0100 Subject: [PATCH] Disable HMX and a few other build hacks --- .../ggml-hexagon/htp/cmake-toolchain.cmake | 2 +- ggml/src/ggml-hexagon/htp/flash-attn-ops.c | 612 +----------------- ggml/src/ggml-hexagon/htp/hex-profile.h | 2 +- ggml/src/ggml-hexagon/htp/hmx-fa-kernels.h | 6 +- .../ggml-hexagon/htp/hmx-mm-kernels-tiled.h | 6 + ggml/src/ggml-hexagon/htp/hvx-div.h | 4 +- ggml/src/ggml-hexagon/htp/matmul-ops.c | 20 +- 7 files changed, 31 insertions(+), 621 deletions(-) diff --git a/ggml/src/ggml-hexagon/htp/cmake-toolchain.cmake b/ggml/src/ggml-hexagon/htp/cmake-toolchain.cmake index 3eff2a398..89307b857 100644 --- a/ggml/src/ggml-hexagon/htp/cmake-toolchain.cmake +++ b/ggml/src/ggml-hexagon/htp/cmake-toolchain.cmake @@ -95,7 +95,7 @@ if (NOT NO_WRAP_MEM_API) set(WRAP_MEMALIGN -Wl,--wrap=memalign) endif() -set(ARCH_FLAGS "-mcpu=${V_ARCH} -m${V_ARCH} -mhvx=${V_ARCH} -mhmx") +set(ARCH_FLAGS "-mcpu=${V_ARCH} -m${V_ARCH} -mhvx=${V_ARCH} -mhvx-ieee-fp -Wno-int-conversion") set(PIC_SHARED_LD_FLAGS ${ARCH_FLAGS} diff --git a/ggml/src/ggml-hexagon/htp/flash-attn-ops.c b/ggml/src/ggml-hexagon/htp/flash-attn-ops.c index a1d15fc7a..736cbec08 100644 --- a/ggml/src/ggml-hexagon/htp/flash-attn-ops.c +++ b/ggml/src/ggml-hexagon/htp/flash-attn-ops.c @@ -15,8 +15,6 @@ #include "hex-dma.h" #include "hex-fastdiv.h" #include "hex-profile.h" -#include "hmx-queue.h" -#include "hmx-utils.h" #include "hvx-utils.h" #include "hvx-dump.h" #include "hvx-copy.h" @@ -468,41 +466,6 @@ typedef struct { uint32_t rows_per_t; } fa_k_int_args_t; -static void fa_k_interleave_thread(unsigned int n, unsigned int i, void * data) { - fa_k_int_args_t * args = (fa_k_int_args_t *) data; - struct hmx_fa_context * factx = args->factx; - - const uint32_t total_rows = args->kv_rows; - const uint32_t rows_per_t = args->rows_per_t; - const uint32_t start = i * rows_per_t; - const uint32_t end = (uint32_t) hex_smin(start + rows_per_t, total_rows); - - if (start >= total_rows) { - return; - } - - struct htp_thread_trace * tr = factx->octx->ctx ? &factx->octx->ctx->trace[i] : NULL; - htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_K_PREP, (uint16_t) (args->kv_start + start)); - hmx_interleave_rows_to_tiles(factx->vtcm_k_tiles, factx->vtcm_k_fp16[args->buf_idx], total_rows, factx->DK, - args->src_stride, start, end); - htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_FA_K_PREP, (uint16_t) (args->kv_start + start)); -} - -static void fa_phase_k_interleave(struct hmx_fa_context * factx, uint32_t kv_rows, size_t src_stride, size_t buf_idx, uint32_t kv_start) { - worker_pool_context_t wp = factx->octx->ctx->worker_pool; - uint32_t n = 1; - if (factx->n_threads > 1 && kv_rows >= factx->n_threads * 2) { - n = factx->n_threads; - } - uint32_t rows_per_t = hex_align_up(hmx_ceil_div(kv_rows, n), 2); - fa_k_int_args_t args = { factx, kv_rows, src_stride, buf_idx, kv_start, rows_per_t }; - if (n > 1) { - worker_pool_run_func(wp, fa_k_interleave_thread, &args, n); - } else { - fa_k_interleave_thread(1, 0, &args); - } -} - typedef struct { struct hmx_fa_context * factx; uint32_t kv_rows; @@ -513,6 +476,7 @@ typedef struct { uint32_t rows_per_t; } fa_v_int_args_t; +#if 0 static void fa_v_interleave_thread(unsigned int n, unsigned int i, void * data) { fa_v_int_args_t * args = (fa_v_int_args_t *) data; struct hmx_fa_context * factx = args->factx; @@ -554,6 +518,7 @@ static void fa_phase_v_interleave(struct hmx_fa_context * factx, fa_v_interleave_thread(1, 0, &args); } } +#endif typedef struct { struct hmx_fa_context * factx; @@ -1318,31 +1283,6 @@ typedef struct { uint8_t * hmx_scales; } hmx_fa_qk_job_t; -static void hmx_fa_qk_dot_worker(void * data) { - hmx_fa_qk_job_t * job = (hmx_fa_qk_job_t *) data; - const size_t n_row_tiles = job->n_row_tiles; - const size_t n_col_tiles = job->n_col_tiles; - const size_t n_dot_tiles = job->n_dot_tiles; - const size_t n_tiles_per_bc = job->n_tiles_per_bc; - const __fp16 * restrict q_tiles = job->q_tiles; - const __fp16 * restrict k_tiles = job->k_tiles; - __fp16 * restrict s_tiles = job->s_tiles; - __builtin_assume(n_row_tiles > 0); - __builtin_assume(n_col_tiles > 0); - __builtin_assume(n_dot_tiles > 0); - - Q6_bias_mxmem2_A((void *) job->hmx_scales); - for (size_t r = 0; r < n_row_tiles; ++r) { - for (size_t c = 0; c < n_col_tiles; ++c) { - const __fp16 * row_tiles = q_tiles + r * HMX_FP16_TILE_N_ROWS * n_dot_tiles * HMX_FP16_TILE_N_COLS; - const __fp16 * col_tiles = k_tiles + c * HMX_FP16_TILE_N_COLS * n_dot_tiles * HMX_FP16_TILE_N_COLS; - __fp16 * out_tile = s_tiles + (r * n_tiles_per_bc + c) * HMX_FP16_TILE_N_ELMS; - - hmx_fa_qk_dot_tile(row_tiles, col_tiles, out_tile, n_dot_tiles); - } - } -} - typedef struct { __fp16 * o_curr; const __fp16 * o_prev; @@ -1357,37 +1297,6 @@ typedef struct { size_t DV; } hmx_fa_o_update_job_t; -static void hmx_fa_o_update_worker(void * data) { - hmx_fa_o_update_job_t * job = (hmx_fa_o_update_job_t *) data; - const size_t n_row_tiles = job->n_row_tiles; - const size_t n_col_tiles = job->n_col_tiles; - const size_t n_row_tiles_g_br = job->n_row_tiles_g_br; - const size_t n_tiles_per_bc = job->n_tiles_per_bc; - const size_t DV_tiles = job->DV / 32; - const __fp16 * restrict d_tiles = job->d_tiles; - const __fp16 * restrict p_tiles = job->p_tiles; - const __fp16 * restrict v_tiles = job->v_tiles; - const __fp16 * restrict o_prev = job->o_prev; - __fp16 * restrict o_curr = job->o_curr; - __builtin_assume(n_row_tiles > 0); - __builtin_assume(n_col_tiles > 0); - __builtin_assume(DV_tiles > 0); - - Q6_bias_mxmem2_A((void *) job->hmx_scales); - for (size_t r = 0; r < n_row_tiles; ++r) { - for (size_t c = 0; c < DV_tiles; ++c) { - // D[r,r] @ O_prev[r,c] — only the diagonal tile - const __fp16 * d_diag = d_tiles + r * (n_row_tiles_g_br + 1) * HMX_FP16_TILE_N_ELMS; - const __fp16 * o_rc = o_prev + (c * n_row_tiles_g_br + r) * HMX_FP16_TILE_N_ELMS; - const __fp16 * p_tile_in = p_tiles + (r * n_tiles_per_bc) * HMX_FP16_TILE_N_ELMS; - const __fp16 * v_tile_in = v_tiles + (c * n_tiles_per_bc) * HMX_FP16_TILE_N_ELMS; - __fp16 * o_tile_out = o_curr + (c * n_row_tiles_g_br + r) * HMX_FP16_TILE_N_ELMS; - - hmx_fa_o_update_tile(d_diag, o_rc, p_tile_in, v_tile_in, o_tile_out, n_col_tiles); - } - } -} - typedef struct { __fp16 * o_curr; // output (row-major tile layout) const __fp16 * o_prev; // input (column-major tile layout) @@ -1398,29 +1307,6 @@ typedef struct { size_t DV; } hmx_fa_o_norm_job_t; -static void hmx_fa_o_norm_worker(void * data) { - hmx_fa_o_norm_job_t * job = (hmx_fa_o_norm_job_t *) data; - const size_t n_row_tiles = job->n_row_tiles; - const size_t n_row_tiles_g_br = job->n_row_tiles_g_br; - const size_t DV_tiles = job->DV / 32; - const __fp16 * restrict d_tiles = job->d_tiles; - const __fp16 * restrict o_prev = job->o_prev; - __fp16 * restrict o_curr = job->o_curr; - __builtin_assume(n_row_tiles > 0); - __builtin_assume(DV_tiles > 0); - - Q6_bias_mxmem2_A((void *) job->hmx_scales); - for (size_t r = 0; r < n_row_tiles; ++r) { - for (size_t c = 0; c < DV_tiles; ++c) { - const __fp16 * d_diag = d_tiles + r * (n_row_tiles_g_br + 1) * HMX_FP16_TILE_N_ELMS; - const __fp16 * o_rc = o_prev + (c * n_row_tiles_g_br + r) * HMX_FP16_TILE_N_ELMS; - __fp16 * o_out = o_curr + (r * DV_tiles + c) * HMX_FP16_TILE_N_ELMS; - - hmx_fa_o_norm_tile(d_diag, o_rc, o_out); - } - } -} - // Populate per-GQA-row ALiBi slopes for a given KV head. static __attribute__((noinline)) void fa_compute_slopes( const struct hmx_fa_context * factx, @@ -1513,500 +1399,6 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { return HTP_STATUS_NO_SUPPORT; } - // Dimensions - const uint32_t neq0 = q->ne[0]; // head_dim (DK) - const uint32_t neq1 = q->ne[1]; // n_tokens - const uint32_t neq2 = q->ne[2]; // n_heads - const uint32_t neq3 = q->ne[3]; // n_seqs - - const uint32_t nek0 = k->ne[0]; // head_dim - const uint32_t nek1 = k->ne[1]; // kv_len - - const uint32_t nev0 = v->ne[0]; // head_dim (DV) - - const uint32_t DK = neq0; - const uint32_t DV = nev0; - - // HMX requires head_dim to be multiple of 32 - if (DK % 32 != 0 || DV % 32 != 0) { - return HTP_STATUS_NO_SUPPORT; - } - - const struct htp_fa_kernel_params * kparams = (const struct htp_fa_kernel_params *) octx->kernel_params; - const uint32_t n_kv_heads = k->ne[2]; - - // ======== Build context ======== - struct hmx_fa_context factx; - memset(&factx, 0, sizeof(factx)); - factx.octx = octx; - factx.sinks = octx->src[4]; // NULL if this op has no attention sinks - factx.n_threads = kparams->n_threads; - factx.DK = DK; - factx.DV = DV; - factx.n_kv = nek1; - factx.n_kv_heads = n_kv_heads; - factx.n_heads = neq2; - factx.G = kparams->G; - factx.div_G = kparams->u.hmx.div_G; - factx.neq1 = neq1; - factx.Br = kparams->Br; - factx.Bc = kparams->Bc; - factx.g_br = kparams->u.hmx.g_br; - factx.n_kv_blocks = kparams->n_kv_blocks; - factx.is_q_fp32 = (kparams->is_q_fp32 != 0); - factx.is_dst_fp32 = (kparams->is_dst_fp32 != 0); - factx.pipeline = (kparams->u.hmx.pipeline != 0); - factx.mask_broadcast = (kparams->u.hmx.mask_broadcast != 0); - if (mask) { - factx.src3_div2 = kparams->src3_div2; - factx.src3_div3 = kparams->src3_div3; - } - - if (kparams->logit_softcap == 0.0f) { - factx.scale = (__fp16) (kparams->scale * EXP_LOG2E_F); // log2(e) - } else { - factx.scale = (__fp16) kparams->scale; - } - factx.max_bias = kparams->max_bias; - factx.logit_softcap = (__fp16) (kparams->logit_softcap * EXP_LOG2E_F); - - factx.n_head_log2 = kparams->n_head_log2; - factx.m0 = kparams->m0; - factx.m1 = kparams->m1; - - const uint32_t Br = factx.Br; - const uint32_t Bc = factx.Bc; - const uint32_t g_br = factx.g_br; - const bool pipeline = factx.pipeline; - const uint32_t n_threads = factx.n_threads; - const uint32_t G = factx.G; - - // ======== VTCM allocation (GQA-aware) ======== - const size_t size_k_row = DK * sizeof(__fp16); - const size_t size_v_row = DV * sizeof(__fp16); - const size_t size_k_row_padded = hex_round_up(size_k_row, 128); - const size_t size_v_row_padded = hex_round_up(size_v_row, 128); - - const size_t q_tile_bytes = hex_align_up(g_br * DK * sizeof(__fp16), 4096); - const size_t o_tile_bytes = hex_align_up(g_br * DV * sizeof(__fp16), 4096); - const size_t k_dma_bytes = hex_align_up(Bc * size_k_row_padded, 4096); - const size_t v_dma_bytes = hex_align_up(Bc * size_v_row_padded, 4096); - const size_t k_tile_bytes = hex_align_up(Bc * DK * sizeof(__fp16), 4096); - const size_t v_tile_bytes = hex_align_up(Bc * DV * sizeof(__fp16), 4096); - const size_t s_tile_bytes = hex_align_up(g_br * Bc * sizeof(__fp16), 4096); - const size_t d_tile_bytes = hex_align_up(g_br * g_br * sizeof(__fp16), 4096); - const size_t col_vec_bytes = hex_align_up(g_br * sizeof(float), 256); - const size_t row_vec_bytes = hex_align_up(Bc * sizeof(__fp16), 256); - const size_t m_line_bytes = hex_align_up(Bc * sizeof(__fp16), 128); - const size_t m_buf_bytes = hex_align_up(Br * m_line_bytes, 4096) * HMX_FA_DMA_CACHE_SIZE; - const size_t slopes_bytes = hex_align_up(g_br * sizeof(__fp16), 128); - - uint8_t * vtcm_cur = ctx->vtcm_base; - - factx.vtcm_q_tiles = (__fp16 *) vtcm_seq_alloc(&vtcm_cur, q_tile_bytes); - factx.vtcm_o_tiles[0] = (__fp16 *) vtcm_seq_alloc(&vtcm_cur, o_tile_bytes); - factx.vtcm_o_tiles[1] = (__fp16 *) vtcm_seq_alloc(&vtcm_cur, o_tile_bytes); - factx.vtcm_k_fp16[0] = (__fp16 *) vtcm_seq_alloc(&vtcm_cur, k_dma_bytes); - factx.vtcm_k_fp16[1] = (__fp16 *) vtcm_seq_alloc(&vtcm_cur, k_dma_bytes); - factx.vtcm_v_fp16[0] = (__fp16 *) vtcm_seq_alloc(&vtcm_cur, v_dma_bytes); - factx.vtcm_v_fp16[1] = (__fp16 *) vtcm_seq_alloc(&vtcm_cur, v_dma_bytes); - factx.vtcm_k_tiles = (__fp16 *) vtcm_seq_alloc(&vtcm_cur, k_tile_bytes); - factx.vtcm_v_tiles[0] = (__fp16 *) vtcm_seq_alloc(&vtcm_cur, v_tile_bytes); - if (pipeline) { - factx.vtcm_v_tiles[1] = (__fp16 *) vtcm_seq_alloc(&vtcm_cur, v_tile_bytes); - } else { - factx.vtcm_v_tiles[1] = NULL; - } - factx.vtcm_s_tiles = (__fp16 *) vtcm_seq_alloc(&vtcm_cur, s_tile_bytes); - factx.vtcm_p_tiles = (__fp16 *) vtcm_seq_alloc(&vtcm_cur, s_tile_bytes); - factx.vtcm_d_tiles = (__fp16 *) vtcm_seq_alloc(&vtcm_cur, d_tile_bytes); - factx.vtcm_m_vec = (HVX_Vector *) vtcm_seq_alloc(&vtcm_cur, col_vec_bytes); - factx.vtcm_l_vec = (HVX_Vector *) vtcm_seq_alloc(&vtcm_cur, col_vec_bytes); - factx.vtcm_s_rowmax = (HVX_Vector *) vtcm_seq_alloc(&vtcm_cur, col_vec_bytes); - factx.vtcm_p_rowsum = (HVX_Vector *) vtcm_seq_alloc(&vtcm_cur, col_vec_bytes); - factx.vtcm_row_bufs = (HVX_Vector *) vtcm_seq_alloc(&vtcm_cur, row_vec_bytes * 2 * n_threads); - factx.row_buf_stride = row_vec_bytes / sizeof(HVX_Vector); - factx.vtcm_hmx_scales_id = vtcm_seq_alloc(&vtcm_cur, 256); - factx.vtcm_hmx_scales_qk = vtcm_seq_alloc(&vtcm_cur, 256); - factx.vtcm_mask_buf = (__fp16 *) vtcm_seq_alloc(&vtcm_cur, m_buf_bytes); - factx.mask_buf_row_stride = m_line_bytes / sizeof(__fp16); - factx.vtcm_slopes = (__fp16 *) vtcm_seq_alloc(&vtcm_cur, slopes_bytes); - - dma_cache_init(&factx.m_cache, (uint8_t *) factx.vtcm_mask_buf, hex_align_up(Br * m_line_bytes, 4096), HMX_FA_DMA_CACHE_SIZE); - - if ((size_t) (vtcm_cur - ctx->vtcm_base) > ctx->vtcm_size) { - return HTP_STATUS_VTCM_TOO_SMALL; - } - - // ======== Initialize HMX output scales ======== - hmx_init_column_scales(factx.vtcm_hmx_scales_id, Q6_V_vsplat_R(0x3c00)); // 1.0 - hmx_init_column_scales(factx.vtcm_hmx_scales_qk, hvx_vec_splat_f16(factx.scale)); - - // ======== Skip compute if profiling ======== - if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) { - return HTP_STATUS_OK; - } - - // ======== DMA setup ======== - dma_queue * const dma = ctx->dma[0]; - - const size_t n_row_tiles_g_br = g_br / HMX_FP16_TILE_N_ROWS; - const size_t n_tiles_per_bc = Bc / HMX_FP16_TILE_N_COLS; - - const size_t qo_element_size = factx.is_q_fp32 ? sizeof(float) : sizeof(__fp16); - - // ======== HMX lock strategy ======== - if (!factx.pipeline) { - HAP_compute_res_hmx_lock(ctx->vtcm_rctx); - } - - // ======== Reusable job descriptors for pipeline ======== - hmx_fa_qk_job_t qk_job; - hmx_fa_o_update_job_t ou_job; - hmx_fa_o_norm_job_t on_job; - - // ======== Main loop ======== - for (uint32_t ib3 = 0; ib3 < neq3; ++ib3) { - const uint32_t im3 = mask ? fastmodulo(ib3, mask->ne[3], &factx.src3_div3) : 0; - for (uint32_t q_start = 0; q_start < neq1; q_start += Br) { - const uint32_t n_q_rows = hex_smin(Br, neq1 - q_start); - const size_t n_rows_g = n_q_rows * G; - const size_t g_br_actual = hex_align_up(n_rows_g, HMX_FP16_TILE_N_ROWS); - const size_t n_row_tiles = g_br_actual / HMX_FP16_TILE_N_ROWS; - - for (uint32_t kv_head = 0; kv_head < n_kv_heads; ++kv_head) { - const uint32_t ik2 = kv_head; - const uint32_t ik3 = ib3 / (neq3 / k->ne[3]); - const uint32_t iv2 = kv_head; - const uint32_t iv3 = ib3 / (neq3 / v->ne[3]); - - // Prefetch first KV block - if (factx.n_kv_blocks > 0) { - const uint32_t kv_rows0 = hex_smin(Bc, nek1); - - const uint8_t * k_src = (const uint8_t *) k->data + ik2 * k->nb[2] + ik3 * k->nb[3]; - dma_queue_push(dma, dma_make_ptr(factx.vtcm_k_fp16[0], k_src), size_k_row_padded, k->nb[1], - size_k_row, kv_rows0); - - const uint8_t * v_src = (const uint8_t *) v->data + iv2 * v->nb[2] + iv3 * v->nb[3]; - dma_queue_push(dma, dma_make_ptr(factx.vtcm_v_fp16[0], v_src), size_v_row_padded, v->nb[1], - size_v_row, kv_rows0); - } - - // ---- Load Q block & Initialize per-block state ---- - fa_phase_q_load(&factx, q, q_start, kv_head, ib3, n_rows_g); - - __fp16 * o_tile_prev = factx.vtcm_o_tiles[0]; - __fp16 * o_tile_curr = factx.vtcm_o_tiles[1]; - - // ---- KV block loop with DMA double-buffering ---- - size_t buf_idx = 0; - - htp_trace_event_start(tr_hvx, HTP_TRACE_EVT_HVX_A_PREP, (uint16_t) q_start); - fa_compute_slopes(&factx, kv_head, n_rows_g); - htp_trace_event_stop(tr_hvx, HTP_TRACE_EVT_HVX_A_PREP, (uint16_t) q_start); - - const size_t k_src_stride = size_k_row_padded / sizeof(__fp16); - const size_t v_src_stride = size_v_row_padded / sizeof(__fp16); - - if (factx.pipeline) { - // ================================================================== - // Pipeline path - // ================================================================== - struct hmx_queue * hmx_q = ctx->hmx_queue; - - for (uint32_t kv_blk = 0; kv_blk < factx.n_kv_blocks; ++kv_blk) { - const uint32_t kv_start = kv_blk * Bc; - const uint32_t kv_rows = hex_smin(Bc, nek1 - kv_start); - const size_t n_col_tiles = hmx_ceil_div(kv_rows, HMX_FP16_TILE_N_COLS); - - // Push mask DMA - if (mask) { - if (__builtin_expect(factx.mask_broadcast, true)) { - const uint8_t * ms_src = (const uint8_t *) mask->data + q_start * mask->nb[1] + im3 * mask->nb[3] + kv_start * sizeof(__fp16); - dma_cache_push(dma, &factx.m_cache, ms_src, m_line_bytes, mask->nb[1], kv_rows * sizeof(__fp16), n_q_rows); - } else { - fa_push_mask_dma_gqa(dma, mask, q_start, im3, kv_start, kv_head, G, m_line_bytes, kv_rows, n_q_rows, &factx); - } - } - - // Wait for current KV DMA - dma_queue_pop(dma); // K - dma_queue_pop(dma); // V - - // ---- Phase 1: K_int ---- - if (kv_blk > 0) { - ou_job.o_curr = o_tile_curr; - ou_job.o_prev = o_tile_prev; - ou_job.p_tiles = factx.vtcm_p_tiles; - ou_job.v_tiles = factx.vtcm_v_tiles[1 - buf_idx]; - ou_job.d_tiles = factx.vtcm_d_tiles; - ou_job.hmx_scales = factx.vtcm_hmx_scales_id; - ou_job.n_row_tiles = n_row_tiles; - ou_job.n_col_tiles = hmx_ceil_div(hex_smin(Bc, nek1 - (kv_blk - 1) * Bc), HMX_FP16_TILE_N_COLS); - ou_job.n_row_tiles_g_br = n_row_tiles_g_br; - ou_job.n_tiles_per_bc = n_tiles_per_bc; - ou_job.DV = DV; - hmx_queue_push(hmx_q, hmx_queue_make_desc(hmx_fa_o_update_worker, &ou_job)); - } - fa_phase_k_interleave(&factx, kv_rows, k_src_stride, buf_idx, kv_start); - - // ---- Phase 2: qk_dot ---- - qk_job.q_tiles = factx.vtcm_q_tiles; - qk_job.k_tiles = factx.vtcm_k_tiles; - qk_job.s_tiles = factx.vtcm_s_tiles; - qk_job.n_row_tiles = n_row_tiles; - qk_job.n_col_tiles = n_col_tiles; - qk_job.n_dot_tiles = DK / 32; - qk_job.n_tiles_per_bc = n_tiles_per_bc; - qk_job.hmx_scales = factx.vtcm_hmx_scales_qk; - hmx_queue_push(hmx_q, hmx_queue_make_desc(hmx_fa_qk_dot_worker, &qk_job)); - - if (kv_blk + 1 < factx.n_kv_blocks) { - const uint32_t prefetch_start = (kv_blk + 1) * Bc; - const uint32_t prefetch_rows = hex_smin(Bc, nek1 - prefetch_start); - const size_t prefetch_buf = 1 - buf_idx; - const uint8_t * k_prefetch_src = (const uint8_t *) k->data + prefetch_start * k->nb[1] + ik2 * k->nb[2] + ik3 * k->nb[3]; - dma_queue_push(dma, dma_make_ptr(factx.vtcm_k_fp16[prefetch_buf], k_prefetch_src), size_k_row_padded, k->nb[1], size_k_row, prefetch_rows); - const uint8_t * v_prefetch_src = (const uint8_t *) v->data + prefetch_start * v->nb[1] + iv2 * v->nb[2] + iv3 * v->nb[3]; - dma_queue_push(dma, dma_make_ptr(factx.vtcm_v_fp16[prefetch_buf], v_prefetch_src), size_v_row_padded, v->nb[1], size_v_row, prefetch_rows); - } - fa_phase_v_interleave(&factx, kv_rows, v_src_stride, buf_idx, n_tiles_per_bc, kv_start); - - if (kv_blk > 0) { - hmx_queue_pop(hmx_q); - hex_swap_ptr((void **) &o_tile_curr, (void **) &o_tile_prev); - } - - hmx_queue_pop(hmx_q); - - // ---- Phase 3: softmax + build_D ---- - __fp16 * current_mask_vtcm = NULL; - if (mask) { - if (__builtin_expect(factx.mask_broadcast, true)) { - current_mask_vtcm = (__fp16 *) dma_queue_pop(dma).dst; - } else { - fa_pop_mask_dma_gqa(dma, G); - current_mask_vtcm = factx.vtcm_mask_buf; - } - } - - fa_softmax_args_t sargs; - memset(&sargs, 0, sizeof(sargs)); - sargs.factx = &factx; - sargs.kv_rows = kv_rows; - sargs.n_rows_g = n_rows_g; - sargs.n_col_tiles = n_col_tiles; - sargs.n_tiles_per_bc = n_tiles_per_bc; - sargs.n_row_tiles = n_row_tiles; - sargs.n_row_tiles_g_br = n_row_tiles_g_br; - sargs.Bc = Bc; - sargs.G = G; - sargs.kv_head = kv_head; - sargs.kv_start = kv_start; - sargs.q_start = q_start; - sargs.ib3 = ib3; - sargs.has_alibi = (factx.max_bias != 0.0f); - sargs.mask = mask; - sargs.mask_vtcm = current_mask_vtcm; - sargs.mask_vtcm_row_stride = factx.mask_buf_row_stride; - sargs.slopes = factx.vtcm_slopes; - fa_phase_softmax_and_build_d(&factx, &sargs, n_row_tiles, n_row_tiles_g_br); - - buf_idx = 1 - buf_idx; - } - - // Epilogue - if (factx.n_kv_blocks > 0) { - const uint32_t last_blk = factx.n_kv_blocks - 1; - const size_t last_cols = hmx_ceil_div(hex_smin(Bc, nek1 - last_blk * Bc), HMX_FP16_TILE_N_COLS); - ou_job.o_curr = o_tile_curr; - ou_job.o_prev = o_tile_prev; - ou_job.p_tiles = factx.vtcm_p_tiles; - ou_job.v_tiles = factx.vtcm_v_tiles[1 - buf_idx]; - ou_job.d_tiles = factx.vtcm_d_tiles; - ou_job.hmx_scales = factx.vtcm_hmx_scales_id; - ou_job.n_row_tiles = n_row_tiles; - ou_job.n_col_tiles = last_cols; - ou_job.n_row_tiles_g_br = n_row_tiles_g_br; - ou_job.n_tiles_per_bc = n_tiles_per_bc; - ou_job.DV = DV; - hmx_queue_push(hmx_q, hmx_queue_make_desc(hmx_fa_o_update_worker, &ou_job)); - hmx_queue_pop(hmx_q); - - hex_swap_ptr((void **) &o_tile_curr, (void **) &o_tile_prev); - } - - } else { - // ================================================================== - // Fallback path - // ================================================================== - for (uint32_t kv_blk = 0; kv_blk < factx.n_kv_blocks; ++kv_blk) { - const uint32_t kv_start = kv_blk * Bc; - const uint32_t kv_rows = hex_smin(Bc, nek1 - kv_start); - const size_t n_col_tiles = hmx_ceil_div(kv_rows, HMX_FP16_TILE_N_COLS); - dma_queue_pop(dma); // K - dma_queue_pop(dma); // V - - if (mask) { - if (__builtin_expect(factx.mask_broadcast, true)) { - const uint8_t * ms_src = (const uint8_t *) mask->data + q_start * mask->nb[1] + im3 * mask->nb[3] + kv_start * sizeof(__fp16); - dma_cache_push(dma, &factx.m_cache, ms_src, m_line_bytes, mask->nb[1], kv_rows * sizeof(__fp16), n_q_rows); - } else { - fa_push_mask_dma_gqa(dma, mask, q_start, im3, kv_start, kv_head, G, m_line_bytes, kv_rows, n_q_rows, &factx); - } - } - if (kv_blk + 1 < factx.n_kv_blocks) { - const uint32_t prefetch_start = (kv_blk + 1) * Bc; - const uint32_t prefetch_rows = hex_smin(Bc, nek1 - prefetch_start); - const size_t prefetch_buf = 1 - buf_idx; - const uint8_t * k_prefetch_src = (const uint8_t *) k->data + prefetch_start * k->nb[1] + ik2 * k->nb[2] + ik3 * k->nb[3]; - dma_queue_push(dma, dma_make_ptr(factx.vtcm_k_fp16[prefetch_buf], k_prefetch_src), size_k_row_padded, k->nb[1], size_k_row, prefetch_rows); - const uint8_t * v_prefetch_src = (const uint8_t *) v->data + prefetch_start * v->nb[1] + iv2 * v->nb[2] + iv3 * v->nb[3]; - dma_queue_push(dma, dma_make_ptr(factx.vtcm_v_fp16[prefetch_buf], v_prefetch_src), size_v_row_padded, v->nb[1], size_v_row, prefetch_rows); - } - fa_phase_k_interleave(&factx, kv_rows, k_src_stride, buf_idx, kv_start); - - { - const size_t n_dot_tiles = (size_t) (DK / 32); - const __fp16 * restrict q_base = factx.vtcm_q_tiles; - const __fp16 * restrict k_base = factx.vtcm_k_tiles; - __fp16 * restrict s_base = factx.vtcm_s_tiles; - __builtin_assume(n_row_tiles > 0); - __builtin_assume(n_col_tiles > 0); - __builtin_assume(n_dot_tiles > 0); - - htp_trace_event_start(tr_hmx, HTP_TRACE_EVT_HMX_COMP, (uint16_t) q_start); - Q6_bias_mxmem2_A((void *) factx.vtcm_hmx_scales_qk); - for (size_t r = 0; r < n_row_tiles; ++r) { - for (size_t c = 0; c < n_col_tiles; ++c) { - const __fp16 * row_tiles = q_base + r * HMX_FP16_TILE_N_ROWS * DK; - const __fp16 * col_tiles = k_base + c * HMX_FP16_TILE_N_COLS * DK; - __fp16 * out_tile = s_base + (r * n_tiles_per_bc + c) * HMX_FP16_TILE_N_ELMS; - - hmx_fa_qk_dot_tile(row_tiles, col_tiles, out_tile, n_dot_tiles); - } - } - htp_trace_event_stop(tr_hmx, HTP_TRACE_EVT_HMX_COMP, (uint16_t) q_start); - } - - // ---- Phase 3: softmax + build_D ---- - __fp16 * current_mask_vtcm = NULL; - if (mask) { - if (__builtin_expect(factx.mask_broadcast, true)) { - current_mask_vtcm = (__fp16 *) dma_queue_pop(dma).dst; - } else { - fa_pop_mask_dma_gqa(dma, G); - current_mask_vtcm = factx.vtcm_mask_buf; - } - } - - fa_softmax_args_t sargs; - memset(&sargs, 0, sizeof(sargs)); - sargs.factx = &factx; - sargs.kv_rows = kv_rows; - sargs.n_rows_g = n_rows_g; - sargs.n_col_tiles = n_col_tiles; - sargs.n_tiles_per_bc = n_tiles_per_bc; - sargs.n_row_tiles = n_row_tiles; - sargs.n_row_tiles_g_br = n_row_tiles_g_br; - sargs.Bc = Bc; - sargs.G = G; - sargs.kv_head = kv_head; - sargs.kv_start = kv_start; - sargs.q_start = q_start; - sargs.ib3 = ib3; - sargs.has_alibi = (factx.max_bias != 0.0f); - sargs.mask = mask; - sargs.mask_vtcm = current_mask_vtcm; - sargs.mask_vtcm_row_stride = factx.mask_buf_row_stride; - sargs.slopes = factx.vtcm_slopes; - fa_phase_softmax_and_build_d(&factx, &sargs, n_row_tiles, n_row_tiles_g_br); - fa_phase_v_interleave(&factx, kv_rows, v_src_stride, buf_idx, n_tiles_per_bc, kv_start); - - { - const size_t DV_tiles = (size_t) (DV / 32); - const __fp16 * restrict d_base = factx.vtcm_d_tiles; - const __fp16 * restrict p_base = factx.vtcm_p_tiles; - const __fp16 * restrict v_base = factx.vtcm_v_tiles[0]; - const __fp16 * restrict op_base = o_tile_prev; - __fp16 * restrict oc_base = o_tile_curr; - __builtin_assume(n_row_tiles > 0); - __builtin_assume(n_col_tiles > 0); - __builtin_assume(DV_tiles > 0); - - htp_trace_event_start(tr_hmx, HTP_TRACE_EVT_HMX_COMP, (uint16_t) q_start); - Q6_bias_mxmem2_A((void *) factx.vtcm_hmx_scales_id); - for (size_t r = 0; r < n_row_tiles; ++r) { - for (size_t c = 0; c < DV_tiles; ++c) { - const __fp16 * d_diag = d_base + r * (n_row_tiles_g_br + 1) * HMX_FP16_TILE_N_ELMS; - const __fp16 * o_rc = op_base + (c * n_row_tiles_g_br + r) * HMX_FP16_TILE_N_ELMS; - const __fp16 * p_tile_in = p_base + (r * n_tiles_per_bc) * HMX_FP16_TILE_N_ELMS; - const __fp16 * v_tile_in = v_base + (c * n_tiles_per_bc) * HMX_FP16_TILE_N_ELMS; - __fp16 * o_tile_out = oc_base + (c * n_row_tiles_g_br + r) * HMX_FP16_TILE_N_ELMS; - - hmx_fa_o_update_tile(d_diag, o_rc, p_tile_in, v_tile_in, o_tile_out, n_col_tiles); - } - } - htp_trace_event_stop(tr_hmx, HTP_TRACE_EVT_HMX_COMP, (uint16_t) q_start); - hex_swap_ptr((void **) &o_tile_curr, (void **) &o_tile_prev); - } - - buf_idx = 1 - buf_idx; - } - } - - // ---- Final normalization ---- - { - htp_trace_event_start(tr_hvx, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) q_start); - fa_build_d_diag_inv_l(&factx, n_row_tiles, n_row_tiles_g_br); - htp_trace_event_stop(tr_hvx, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) q_start); - - if (factx.pipeline) { - on_job.o_curr = o_tile_curr; - on_job.o_prev = o_tile_prev; - on_job.d_tiles = factx.vtcm_d_tiles; - on_job.hmx_scales = factx.vtcm_hmx_scales_id; - on_job.n_row_tiles = n_row_tiles; - on_job.n_row_tiles_g_br = n_row_tiles_g_br; - on_job.DV = DV; - hmx_queue_push(ctx->hmx_queue, hmx_queue_make_desc(hmx_fa_o_norm_worker, &on_job)); - hmx_queue_pop(ctx->hmx_queue); - } else { - const size_t DV_tiles = (size_t) (DV / 32); - const __fp16 * restrict d_base = factx.vtcm_d_tiles; - const __fp16 * restrict op_base = o_tile_prev; - __fp16 * restrict oc_base = o_tile_curr; - __builtin_assume(n_row_tiles > 0); - __builtin_assume(DV_tiles > 0); - - htp_trace_event_start(tr_hmx, HTP_TRACE_EVT_HMX_COMP, (uint16_t) q_start); - Q6_bias_mxmem2_A((void *) factx.vtcm_hmx_scales_id); - for (size_t r = 0; r < n_row_tiles; ++r) { - for (size_t c = 0; c < DV_tiles; ++c) { - const __fp16 * d_diag = d_base + r * (n_row_tiles_g_br + 1) * HMX_FP16_TILE_N_ELMS; - const __fp16 * o_rc = op_base + (c * n_row_tiles_g_br + r) * HMX_FP16_TILE_N_ELMS; - __fp16 * o_out = oc_base + (r * DV_tiles + c) * HMX_FP16_TILE_N_ELMS; - - hmx_fa_o_norm_tile(d_diag, o_rc, o_out); - } - } - htp_trace_event_stop(tr_hmx, HTP_TRACE_EVT_HMX_COMP, (uint16_t) q_start); - } - } - - // ---- Store O block ---- - fa_phase_o_store(&factx, dst, o_tile_curr, q_start, kv_head, ib3, n_rows_g); - } - } - } - - if (factx.pipeline) { - hmx_queue_suspend(ctx->hmx_queue); - } else { - HAP_compute_res_hmx_unlock(ctx->vtcm_rctx); - } - return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/hex-profile.h b/ggml/src/ggml-hexagon/htp/hex-profile.h index 8a37a4a06..cadc53cc6 100644 --- a/ggml/src/ggml-hexagon/htp/hex-profile.h +++ b/ggml/src/ggml-hexagon/htp/hex-profile.h @@ -16,7 +16,7 @@ #endif static inline void hex_get_pmu(uint32_t counters[]) { -#if __HVX_ARCH__ >= 79 +#if 0 asm volatile("%0 = upmucnt0" : "=r"(counters[0])); asm volatile("%0 = upmucnt1" : "=r"(counters[1])); asm volatile("%0 = upmucnt2" : "=r"(counters[2])); diff --git a/ggml/src/ggml-hexagon/htp/hmx-fa-kernels.h b/ggml/src/ggml-hexagon/htp/hmx-fa-kernels.h index 1ab286e1c..e044db98c 100644 --- a/ggml/src/ggml-hexagon/htp/hmx-fa-kernels.h +++ b/ggml/src/ggml-hexagon/htp/hmx-fa-kernels.h @@ -5,7 +5,7 @@ #include #include #include "hvx-utils.h" -#include "hmx-utils.h" +//#include "hmx-utils.h" // HMX-specific parameters, offsets and inner kernels for Flash Attention @@ -45,6 +45,8 @@ static const int16_t d_tile_scatter_offsets[64] __attribute__((aligned(128))) = 0, 0, 0, 0, }; + +#if 0 // Inner HMX tile computation kernels static inline void hmx_fa_qk_dot_tile( @@ -93,4 +95,6 @@ static inline void hmx_fa_o_norm_tile( Q6_mxmem_AR_after_hf(o_out, 0); } +#endif + #endif /* HMX_FA_KERNELS_H */ diff --git a/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h b/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h index 4a0ca7885..300c49e24 100644 --- a/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h +++ b/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h @@ -553,6 +553,7 @@ static void convert_f16_weight_to_fp16_tiles_task( } } + static void quantize_f32_weight_to_fp16_tiles_task( const tiled_dequantize_state_t *state, uint32_t start_tile, uint32_t end_tile) { @@ -606,6 +607,7 @@ static void quantize_f32_weight_to_fp16_tiles_task( // --- End tiled dequantizers --- +#if 0 // requires external HMX lock static void core_dot_chunk_fp16(__fp16 *restrict output, const __fp16 *restrict activation, const __fp16 *restrict weight, const __fp16 *restrict scales, uint32_t n_row_tiles, uint32_t n_col_tiles, uint32_t n_dot_tiles) { @@ -710,6 +712,8 @@ static inline void hmx_matmul_job_init(hmx_matmul_job_t * job, job->n_dot_tiles = n_dot_tiles; } +#endif + // output : fp16 -> f32p static void transfer_output_chunk_fp16_to_fp32( @@ -793,6 +797,7 @@ static void transfer_output_chunk_fp16_to_fp32( } } + typedef struct { const __fp16 *vtcm_src; float *dst; @@ -1339,3 +1344,4 @@ static void transfer_output_chunk_fp16_to_fp32_scattered( } } } + diff --git a/ggml/src/ggml-hexagon/htp/hvx-div.h b/ggml/src/ggml-hexagon/htp/hvx-div.h index 53ee304e7..c28f2a91f 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-div.h +++ b/ggml/src/ggml-hexagon/htp/hvx-div.h @@ -37,7 +37,7 @@ static inline HVX_Vector hvx_div_mul_f16_const_using_f32(HVX_Vector vec1_hf, HVX HVX_Vector div_f32_0 = HVX_OP_MUL_F32(src_to_f32_0, vec2_sf_const); HVX_Vector div_f32_1 = HVX_OP_MUL_F32(src_to_f32_1, vec2_sf_const); -#if __HVX_ARCH__ < 79 +#if __HVX_ARCH__ < 79 || !defined(__HVX_IEEE_FP__) HVX_Vector res = hvx_vec_f32_to_f16(div_f32_0, div_f32_1); #else HVX_Vector res = Q6_Vhf_vcvt_VsfVsf(div_f32_0, div_f32_1); @@ -111,7 +111,7 @@ static inline void hvx_div_scalar_f16_uu(uint8_t * restrict dst, const uint8_t * // Compute div by using hvx_vec_inverse_f32_guard. Requires first by exapnding fp32 to fp16 and convert the result back to fp32. static inline HVX_Vector hvx_vec_div_f16_using_f32(HVX_Vector vec1, HVX_Vector vec2, HVX_Vector f32_nan_inf_mask, HVX_Vector vec_hf_one_1_0) { -#if __HVX_ARCH__ < 79 +#if __HVX_ARCH__ < 79 || !defined(__HVX_IEEE_FP__) // Convert first input to fp32 HVX_VectorPair vec1_to_f32 = Q6_Wqf32_vmpy_VhfVhf(vec1, vec_hf_one_1_0); // *1.0 HVX_Vector vec1_to_f32_0 = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(vec1_to_f32)); diff --git a/ggml/src/ggml-hexagon/htp/matmul-ops.c b/ggml/src/ggml-hexagon/htp/matmul-ops.c index 79c292dc2..93f752e5e 100644 --- a/ggml/src/ggml-hexagon/htp/matmul-ops.c +++ b/ggml/src/ggml-hexagon/htp/matmul-ops.c @@ -2139,6 +2139,7 @@ static void transfer_activation_chunk_threaded( } } +#if 0 static int hmx_mm_2d_f32(struct htp_context *ctx, float *restrict dst, const float *restrict src2, @@ -2370,6 +2371,7 @@ static int hmx_mm_2d_f32(struct htp_context *ctx, return 0; } +#endif static inline int hmx_mm_batch_r2(const hmx_mm_f16_f32_batched_params_t *params) { return params->ne02 > 0 ? params->ne12 / params->ne02 : 1; @@ -2409,6 +2411,7 @@ static inline const float *hmx_mm_src2_batch_ptr(const hmx_mm_f16_f32_batched_pa (size_t) src2_b3 * params->src2_nb3) : NULL; } +#if 0 static int hmx_mm_f16_f32_batched_simple(struct htp_context *ctx, const hmx_mm_f16_f32_batched_params_t *params, int m_chunk, int n_chunk, int pipeline, int n_threads, int act_threads, int vtcm_size) { @@ -2569,6 +2572,7 @@ static int hmx_mm_f16_f32_batched(struct htp_context *ctx, const hmx_mm_f16_f32_ return 0; } +#endif static void transfer_activation_chunk_gathered_threaded( struct htp_context *ctx, @@ -2666,6 +2670,7 @@ static void transfer_output_chunk_scattered_threaded( } } +#if 0 static int hmx_mm_id_2d_f32(struct htp_context *ctx, float *restrict dst, const float *activation, @@ -2875,17 +2880,19 @@ static int hmx_mm_op_matmul(struct htp_ops_context * octx, const struct htp_mm_k } return HTP_STATUS_OK; } +#endif int op_matmul(struct htp_ops_context * octx) { const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; - if (kparams->n_hmx) { - return hmx_mm_op_matmul(octx, kparams); - } + //if (kparams->n_hmx) { + // return hmx_mm_op_matmul(octx, kparams); + //} return hvx_mm_matmul(octx); } +#if 0 static int hmx_mm_op_matmul_id( struct htp_ops_context * octx, struct htp_mm_context * mmctx, @@ -2922,6 +2929,7 @@ static int hmx_mm_op_matmul_id( if (must_free_mapping) free(mapping_buf); return HTP_STATUS_OK; } +#endif static int hvx_mm_matmul_id( struct htp_ops_context * octx, @@ -3105,9 +3113,9 @@ int op_matmul_id(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - if (kparams->n_hmx) { - return hmx_mm_op_matmul_id(octx, mmctx, matrix_row_counts, matrix_rows, mapping_buf, must_free_mapping); - } + //if (kparams->n_hmx) { + // return hmx_mm_op_matmul_id(octx, mmctx, matrix_row_counts, matrix_rows, mapping_buf, must_free_mapping); + //} return hvx_mm_matmul_id(octx, mmctx, src0_row_size_padded, src1_nrows, matmul_id_job_func, mapping_buf, must_free_mapping); } -- 2.53.0