/*
 * Copyright © 2017 Rob Clark <robclark@freedesktop.org>
 * Copyright © 2018 Google, Inc.
 * SPDX-License-Identifier: MIT
 *
 * Authors:
 *    Rob Clark <robclark@freedesktop.org>
 */

#include "pipe/p_state.h"

#include "freedreno_resource.h"
#include "freedreno_state.h"

#include "fd6_image.h"
#include "fd6_pack.h"
#include "fd6_resource.h"
#include "fd6_screen.h"
#include "fd6_texture.h"

static const uint8_t swiz_identity[4] = {PIPE_SWIZZLE_X, PIPE_SWIZZLE_Y,
                                         PIPE_SWIZZLE_Z, PIPE_SWIZZLE_W};

static uint64_t
rsc_iova(struct pipe_resource *prsc, unsigned offset)
{
   if (!prsc)
      return 0;
   return fd_bo_get_iova(fd_resource(prsc)->bo) + offset;
}

template <chip CHIP>
static void
fd6_ssbo_descriptor(struct fd_context *ctx,
                    const struct pipe_shader_buffer *buf, uint32_t *descriptor)
{
   fdl6_buffer_view_init<CHIP>(
      descriptor,
      PIPE_FORMAT_R32_UINT,
      swiz_identity, rsc_iova(buf->buffer, buf->buffer_offset),
      buf->buffer_size);
}

template <chip CHIP>
static void
fd6_image_descriptor(struct fd_context *ctx, const struct pipe_image_view *buf,
                     uint32_t *descriptor)
{
   if (buf->access & PIPE_IMAGE_ACCESS_TEX2D_FROM_BUFFER) {
      struct fdl_view_args args = {
         .iova = rsc_iova(buf->resource, 0),
         .base_miplevel = 0,
         .level_count = 1,
         .base_array_layer = 0,
         .layer_count = 1,
         .swiz = {PIPE_SWIZZLE_X, PIPE_SWIZZLE_Y, PIPE_SWIZZLE_Z,
                  PIPE_SWIZZLE_W},
         .format = buf->format,

         .type = FDL_VIEW_TYPE_2D,
         .chroma_offsets = {FDL_CHROMA_LOCATION_COSITED_EVEN,
                            FDL_CHROMA_LOCATION_COSITED_EVEN},
      };

      struct fdl_layout layout;
      const struct fdl_layout *layouts = &layout;

      fd6_layout_tex2d_from_buf(&layout, ctx->screen->info, buf->format,
                                &buf->u.tex2d_from_buf);

      struct fdl6_view view;
      fdl6_view_init<CHIP>(&view, &layouts, &args,
                           ctx->screen->info->props.has_z24uint_s8uint);

      memcpy(descriptor, view.storage_descriptor, sizeof(view.storage_descriptor));
   } else if (buf->resource->target == PIPE_BUFFER) {
      uint32_t size = fd_clamp_buffer_size(buf->format, buf->u.buf.size,
                                           A4XX_MAX_TEXEL_BUFFER_ELEMENTS_UINT);

      fdl6_buffer_view_init<CHIP>(descriptor, buf->format, swiz_identity,
                                  rsc_iova(buf->resource, buf->u.buf.offset),
                                  size);
   } else {
      struct fdl_view_args args = {
         .iova = rsc_iova(buf->resource, 0),

         .base_miplevel = buf->u.tex.level,
         .level_count = 1,

         .base_array_layer = buf->u.tex.first_layer,
         .layer_count = buf->u.tex.last_layer - buf->u.tex.first_layer + 1,

         .swiz = {PIPE_SWIZZLE_X, PIPE_SWIZZLE_Y, PIPE_SWIZZLE_Z,
                  PIPE_SWIZZLE_W},
         .format = buf->format,

         .type = fdl_type_from_pipe_target(buf->resource->target),
         .chroma_offsets = {FDL_CHROMA_LOCATION_COSITED_EVEN,
                            FDL_CHROMA_LOCATION_COSITED_EVEN},
      };

      /* fdl6_view makes the storage descriptor treat cubes like a 2D array (so
       * you can reference a specific layer), but we need to do that for the
       * texture descriptor as well to get our layer.
       */
      if (args.type == FDL_VIEW_TYPE_CUBE)
         args.type = FDL_VIEW_TYPE_2D;

      struct fdl6_view view;
      struct fd_resource *rsc = fd_resource(buf->resource);
      const struct fdl_layout *layouts[3] = { &rsc->layout, NULL, NULL };
      fdl6_view_init<CHIP>(&view, layouts, &args,
                           ctx->screen->info->props.has_z24uint_s8uint);

      memcpy(descriptor, view.storage_descriptor, sizeof(view.storage_descriptor));
   }
}

static struct fd6_descriptor_set *
descriptor_set(struct fd_context *ctx, mesa_shader_stage shader)
   assert_dt
{
   struct fd6_context *fd6_ctx = fd6_context(ctx);

   if (shader == MESA_SHADER_COMPUTE)
      return &fd6_ctx->cs_descriptor_set;

   unsigned idx = ir3_shader_descriptor_set(shader);
   assert(idx < ARRAY_SIZE(fd6_ctx->descriptor_sets));
   return &fd6_ctx->descriptor_sets[idx];
}

static void
clear_descriptor(struct fd6_descriptor_set *set, unsigned slot)
{
   /* The 2nd dword of the descriptor contains the width and height.
    * so a non-zero value means the slot was previously valid and
    * must be cleared.  We can't leave dangling descriptors as the
    * shader could use variable indexing into the set of UAVs to
    * get at them.  See piglit arb_shader_image_load_store-invalid.
    */
   if (!set->descriptor[slot][1])
      return;

   fd6_descriptor_set_invalidate(set);

   memset(set->descriptor[slot], 0, sizeof(set->descriptor[slot]));
}

template <chip CHIP>
static void
validate_image_descriptor(struct fd_context *ctx, struct fd6_descriptor_set *set,
                          unsigned slot, struct pipe_image_view *img)
{
   struct fd_resource *rsc = fd_resource(img->resource);

   if (!rsc || (rsc->seqno == set->seqno[slot]))
      return;

   fd6_descriptor_set_invalidate(set);

   fd6_image_descriptor<CHIP>(ctx, img, set->descriptor[slot]);
   set->seqno[slot] = rsc->seqno;
}

template <chip CHIP>
static void
validate_buffer_descriptor(struct fd_context *ctx, struct fd6_descriptor_set *set,
                           unsigned slot, struct pipe_shader_buffer *buf)
{
   struct fd_resource *rsc = fd_resource(buf->buffer);

   if (!rsc || (rsc->seqno == set->seqno[slot]))
      return;

   fd6_descriptor_set_invalidate(set);

   fd6_ssbo_descriptor<CHIP>(ctx, buf, set->descriptor[slot]);
   set->seqno[slot] = rsc->seqno;
}

/* Build bindless descriptor state, returns ownership of state reference */
template <chip CHIP>
struct fd_ringbuffer *
fd6_build_bindless_state(struct fd_context *ctx, mesa_shader_stage shader,
                         bool append_fb_read)
{
   struct fd_shaderbuf_stateobj *bufso = &ctx->shaderbuf[shader];
   struct fd_shaderimg_stateobj *imgso = &ctx->shaderimg[shader];
   struct fd6_descriptor_set *set = descriptor_set(ctx, shader);

   fd_cs cs(ctx->batch->submit, 19 * 4);

   /* Don't re-use a previous descriptor set if appending the
    * fb-read descriptor, as that can change across batches.
    * The normal descriptor slots are safe to re-use even if
    * the state is dirtied due to batch flush, but the fb-read
    * slot is not.
    */
   if (unlikely(append_fb_read))
      fd6_descriptor_set_invalidate(set);

   /*
    * Re-validate the descriptor slots, ie. in the case that
    * the resource gets rebound due to use with non-UBWC
    * compatible view format, etc.
    *
    * While we are at it, attach the BOs to the ring.
    */

   u_foreach_bit (b, bufso->enabled_mask) {
      struct pipe_shader_buffer *buf = &bufso->sb[b];
      unsigned idx = b + IR3_BINDLESS_SSBO_OFFSET;
      validate_buffer_descriptor<CHIP>(ctx, set, idx, buf);
   }

   u_foreach_bit (b, imgso->enabled_mask) {
      struct pipe_image_view *img = &imgso->si[b];
      unsigned idx = b + IR3_BINDLESS_IMAGE_OFFSET;
      validate_image_descriptor<CHIP>(ctx, set, idx, img);
   }

   if (!set->bo) {
      set->bo = fd_bo_new(
            ctx->dev, sizeof(set->descriptor),
            /* Use same flags as ringbuffer so hits the same heap,
             * because those will already have the FD_RELOC_DUMP
             * flag set:
             */
            FD_BO_GPUREADONLY | FD_BO_CACHED_COHERENT,
            "%s bindless", _mesa_shader_stage_to_abbrev(shader));
      fd_bo_mark_for_dump(set->bo);

      uint32_t *desc_buf = (uint32_t *)fd_bo_map(set->bo);

      memcpy(desc_buf, set->descriptor, sizeof(set->descriptor));

      if (unlikely(append_fb_read)) {
         /* Reserve A6XX_MAX_RENDER_TARGETS image slots for fb-read */
         unsigned idx = IR3_BINDLESS_DESC_COUNT - 1 - A6XX_MAX_RENDER_TARGETS;

         for (int i = 0; i < ctx->batch->framebuffer.nr_cbufs; i++) {
            /* This is patched with the appropriate descriptor for GMEM or
             * sysmem rendering path in fd6_gmem
             */
            struct fd_cs_patch patch = {
               .cs = &desc_buf[(idx + i) * FDL6_TEX_CONST_DWORDS],
               .val = i,
            };
            util_dynarray_append(&ctx->batch->fb_read_patches, patch);
         }
      }
   }

   /*
    * Build stateobj emitting reg writes to configure the descriptor
    * set and CP_LOAD_STATE packets to preload the state.
    *
    * Note that unless the app is using the max # of SSBOs there will
    * be a gap between the UAV descriptors used for SSBOs and for images,
    * so emit this as two CP_LOAD_STATE packets:
    */

   unsigned idx = ir3_shader_descriptor_set(shader);

   cs.attach_bo(set->bo);

   if (shader == MESA_SHADER_COMPUTE) {
      with_crb (cs, 5) {
         crb.add(SP_UPDATE_CNTL(CHIP,
            .cs_bindless = CHIP == A6XX ? 0x1f : 0xff,
         ));
         crb.add(SP_CS_BINDLESS_BASE_DESCRIPTOR(CHIP,
            idx, .desc_size = BINDLESS_DESCRIPTOR_64B, .bo = set->bo,
         ));

         if (CHIP == A6XX) {
            crb.add(HLSQ_CS_BINDLESS_BASE_DESCRIPTOR(CHIP,
               idx, .desc_size = BINDLESS_DESCRIPTOR_64B, .bo = set->bo,
            ));
         }
      }

      if ((CHIP < A8XX) && bufso->enabled_mask) {
         fd_pkt7(cs, CP_LOAD_STATE6_FRAG, 3)
            .add(CP_LOAD_STATE6_0(
               .dst_off     = IR3_BINDLESS_SSBO_OFFSET,
               .state_type  = ST6_UAV,
               .state_src   = SS6_BINDLESS,
               .state_block = SB6_CS_SHADER,
               .num_unit    = util_last_bit(bufso->enabled_mask),
            ))
            .add(CP_LOAD_STATE6_EXT_SRC_ADDR(
               /* This isn't actually an address: */
               .qword = (idx << 28) |
                  IR3_BINDLESS_SSBO_OFFSET * FDL6_TEX_CONST_DWORDS,
            ));
      }

      if ((CHIP < A8XX) && imgso->enabled_mask) {
         fd_pkt7(cs, CP_LOAD_STATE6_FRAG, 3)
            .add(CP_LOAD_STATE6_0(
               .dst_off     = IR3_BINDLESS_IMAGE_OFFSET,
               .state_type  = ST6_UAV,
               .state_src   = SS6_BINDLESS,
               .state_block = SB6_CS_SHADER,
               .num_unit    = util_last_bit(imgso->enabled_mask),
            ))
            .add(CP_LOAD_STATE6_EXT_SRC_ADDR(
               /* This isn't actually an address: */
               .qword = (idx << 28) |
                  IR3_BINDLESS_IMAGE_OFFSET * FDL6_TEX_CONST_DWORDS,
            ));
      }
   } else {
      with_crb (cs, 5) {
         crb.add(SP_UPDATE_CNTL(CHIP,
            .gfx_bindless = CHIP == A6XX ? 0x1f : 0xff,
         ));
         crb.add(SP_GFX_BINDLESS_BASE_DESCRIPTOR(CHIP,
            idx, .desc_size = BINDLESS_DESCRIPTOR_64B, .bo = set->bo,
         ));
         if (CHIP == A6XX) {
            crb.add(HLSQ_BINDLESS_BASE_DESCRIPTOR(CHIP,
               idx, .desc_size = BINDLESS_DESCRIPTOR_64B, .bo = set->bo,
            ));
         }
      }

      if ((CHIP < A8XX) && bufso->enabled_mask) {
         fd_pkt7(cs, CP_LOAD_STATE6, 3)
            .add(CP_LOAD_STATE6_0(
               .dst_off     = IR3_BINDLESS_SSBO_OFFSET,
               .state_type  = ST6_SHADER,
               .state_src   = SS6_BINDLESS,
               .state_block = SB6_UAV,
               .num_unit    = util_last_bit(bufso->enabled_mask),
            ))
            .add(CP_LOAD_STATE6_EXT_SRC_ADDR(
               /* This isn't actually an address: */
               .qword = (idx << 28) |
                  IR3_BINDLESS_SSBO_OFFSET * FDL6_TEX_CONST_DWORDS,
            ));
      }

      if ((CHIP < A8XX) && imgso->enabled_mask) {
         fd_pkt7(cs, CP_LOAD_STATE6, 3)
            .add(CP_LOAD_STATE6_0(
               .dst_off     = IR3_BINDLESS_IMAGE_OFFSET,
               .state_type  = ST6_SHADER,
               .state_src   = SS6_BINDLESS,
               .state_block = SB6_UAV,
               .num_unit    = util_last_bit(imgso->enabled_mask),
            ))
            .add(CP_LOAD_STATE6_EXT_SRC_ADDR(
               /* This isn't actually an address: */
               .qword = (idx << 28) |
                  IR3_BINDLESS_IMAGE_OFFSET * FDL6_TEX_CONST_DWORDS,
            ));
      }
   }

   return cs;
}
FD_GENX(fd6_build_bindless_state);

template <chip CHIP>
static void
fd6_set_shader_buffers(struct pipe_context *pctx, mesa_shader_stage shader,
                       unsigned start, unsigned count,
                       const struct pipe_shader_buffer *buffers,
                       unsigned writable_bitmask)
   in_dt
{
   struct fd_context *ctx = fd_context(pctx);
   struct fd_shaderbuf_stateobj *so = &ctx->shaderbuf[shader];
   struct fd6_descriptor_set *set = descriptor_set(ctx, shader);

   fd_set_shader_buffers(pctx, shader, start, count, buffers, writable_bitmask);

   for (unsigned i = 0; i < count; i++) {
      unsigned n = i + start;
      unsigned slot = n + IR3_BINDLESS_SSBO_OFFSET;
      struct pipe_shader_buffer *buf = &so->sb[n];

      /* invalidate descriptor: */
      set->seqno[slot] = 0;

      if (!buf->buffer) {
         clear_descriptor(set, slot);
         continue;
      }

      /* update descriptor: */
      validate_buffer_descriptor<CHIP>(ctx, set, slot, buf);
   }
}

template <chip CHIP>
static void
fd6_set_shader_images(struct pipe_context *pctx, mesa_shader_stage shader,
                      unsigned start, unsigned count,
                      unsigned unbind_num_trailing_slots,
                      const struct pipe_image_view *images)
   in_dt
{
   struct fd_context *ctx = fd_context(pctx);
   struct fd_shaderimg_stateobj *so = &ctx->shaderimg[shader];
   struct fd6_descriptor_set *set = descriptor_set(ctx, shader);

   fd_set_shader_images(pctx, shader, start, count, unbind_num_trailing_slots,
                        images);

   for (unsigned i = 0; i < count; i++) {
      unsigned n = i + start;
      unsigned slot = n + IR3_BINDLESS_IMAGE_OFFSET;
      struct pipe_image_view *buf = &so->si[n];

      /* invalidate descriptor: */
      set->seqno[slot] = 0;

      if (!buf->resource) {
         clear_descriptor(set, slot);
         continue;
      }

      struct fd_resource *rsc = fd_resource(buf->resource);

      if (buf->shader_access & (PIPE_IMAGE_ACCESS_COHERENT |
                                PIPE_IMAGE_ACCESS_VOLATILE)) {
         /* UBWC compression cannot be used with coherent/volatile access
          * due to the extra caching (CCU) involved:
          */
         if (rsc->layout.ubwc) {
            bool linear =
                  fd6_check_valid_format(rsc, buf->format) == DEMOTE_TO_LINEAR;

            perf_debug_ctx(ctx,
                           "%" PRSC_FMT ": demoted to %suncompressed due to coherent/volatile use as %s",
                           PRSC_ARGS(&rsc->b.b), linear ? "linear+" : "",
                           util_format_short_name(buf->format));

            fd_resource_uncompress(ctx, rsc, linear);
         }
      } else {
         fd6_validate_format(ctx, rsc, buf->format);
      }

      /* update descriptor: */
      validate_image_descriptor<CHIP>(ctx, set, slot, buf);
   }

   for (unsigned i = 0; i < unbind_num_trailing_slots; i++) {
      unsigned slot = i + start + count + IR3_BINDLESS_IMAGE_OFFSET;

      set->seqno[slot] = 0;
      clear_descriptor(set, slot);
   }
}

template <chip CHIP>
void
fd6_image_init(struct pipe_context *pctx)
{
   pctx->set_shader_buffers = fd6_set_shader_buffers<CHIP>;
   pctx->set_shader_images = fd6_set_shader_images<CHIP>;
}
FD_GENX(fd6_image_init);
